From: Svjatoslav Agejenko Date: Sat, 18 Jul 2026 11:12:38 +0000 (+0300) Subject: initial commit X-Git-Url: http://www2.svjatoslav.eu/gitweb/?a=commitdiff_plain;h=1528b7dd210904b085b54d56de4f963d6061fb52;p=retinue.git initial commit --- 1528b7dd210904b085b54d56de4f963d6061fb52 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..06c3af6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/.venv/ +/.hf-cache/ +__pycache__/ +*.pyc +/.idea +*.html \ No newline at end of file diff --git a/AGENTS.org b/AGENTS.org new file mode 100644 index 0000000..b508afb --- /dev/null +++ b/AGENTS.org @@ -0,0 +1,250 @@ +#+TITLE: AGENTS — Operating guide for Retinue + +* At a glance +:PROPERTIES: +:ID: 94584d56-4303-4aef-ab1c-8757bd860ff1 +:END: + +| Fact | Value | +|------------------+----------------------------------------------------------------| +| Project | Retinue — semantic memory provider plugin for Hermes Agent | +| License | CC0 (see COPYING) | +| Language | Python 3 (PEP 668 host — always use a venv, never system pip) | +| Runtime deps | =model2vec=, =sqlite-vec= | +| Embedding model | =minishlab/potion-multilingual-128M= (~540 MB, cached offline; the repo's unused ONNX export is skipped) | +| Plugin metadata | plugin.yaml (version 1.1.0) | +| Public docs site | https://www.svjatoslav.eu/projects/ (served from Documentation/) | + +* Purpose +:PROPERTIES: +:ID: 67822f95-d295-455b-ac84-fde0aed8b008 +:END: + +Retinue adds persistent, semantic memory to every Hermes Agent +session. It is a Hermes =MemoryProvider= plugin: the LLM stores and +recalls facts through four =retinue_memory_*= tools backed by +model2vec embeddings and sqlite-vec vector search, fully local, +CPU-only, single SQLite file. + +Retinue is developed in this workspace and *deployed* into the active +Hermes home (~/.hermes/plugins/retinue/) by the =Install Retinue= script. +The deployed copy is disposable — this tree is the source of truth. Never +edit =~/.hermes/plugins/retinue/= directly; edit here and re-run the +installer. + +* Layout +:PROPERTIES: +:ID: a8c8c555-d1c1-4f96-b236-be8106e03a9a +:END: + +#+begin_example +retinue/ +├── plugin.yaml # Hermes plugin metadata (name, version, hooks) +├── __init__.py # Plugin entry point: register(ctx) +├── src/retinue/ +│ ├── __init__.py +│ ├── memory.py # MemoryStore: model2vec embed + sqlite-vec CRUD +│ ├── provider.py # RetinueMemoryProvider: MemoryProvider ABC impl +│ └── system_prompt_block.txt # Injected into the Hermes system prompt +├── Install Retinue # Deploy into $HERMES_HOME/plugins/retinue/ +├── test/test_standalone.py # End-to-end MemoryStore + provider tests +├── Documentation/ +│ ├── index.org # Public user documentation (exports to HTML) +│ └── Development/index.org # Architecture & integration documentation +├── Tools/ # Development helpers (filenames contain spaces) +│ ├── Run tests # Standalone smoke test runner (no Hermes needed) +│ ├── Show Retinue memory contents # Debug CLI: dump/search all memories +│ ├── Import Holographic memories # Migrate facts from Holographic memory_store.db +│ ├── Open with IntelliJ IDEA +│ └── Update web site # Export Documentation/*.org to HTML, rsync to www3 +#+end_example + +Created at runtime, never committed (=.gitignore=): +- =.venv/= — project-local venv used by "Tools/Run tests" +- =.hf-cache/= — Hugging Face cache for the embedding model +- =__pycache__/=, =*.pyc=, =.idea/=, =*.html= (HTML exports are regenerated) + +The deployed database lives in the active Hermes home, not here: +=$HERMES_HOME/retinue_memory.db= (single file: SQLite + sqlite-vec +virtual table =memories=). + +* Conventions +:PROPERTIES: +:ID: 237ae2c1-1c42-4572-ab0a-d118b90e52cb +:END: + +** Org-mode documentation + +- Every .org page starts with a file-level =:PROPERTIES:= drawer holding + a unique =:ID:= UUID, then =#+SETUPFILE: ~/.emacs.d/org-styles/html/darksun.theme=, + =#+TITLE:=, =#+LANGUAGE: en=, and the three standard =#+LATEX_HEADER:= + lines (geometry, parskip, hyphenat). Copy the header of + =Documentation/index.org= verbatim for new pages. +- Every heading gets its own =:PROPERTIES:= drawer with a fresh =:ID:= + UUID (and a kebab-case =:CUSTOM_ID:= matching the heading text for + in-page anchors). Generate UUIDs with + =python3 -c "import uuid; print(uuid.uuid4())"=. +- Cross-page links use =[[file:...]]=, e.g. =[[file:Development/][Development]]= + and =[[file:../index.org][Back to main documentation]]=. +- Org tables need a =|----+----|= separator row between header and data, + and a trailing =|= on every row. +- Options line is exactly: + =#+OPTIONS: H:20 num:20= plus =#+OPTIONS: author:nil=. + +** Code + +- ~...~ for inline code spans in org (=~/.hermes=, ="Tools/Run tests"=); + =...= for verbatim tokens (function names, keywords, model IDs). +- Python: stdlib + =model2vec= + =sqlite-vec= only. No new runtime + dependencies without a strong reason — the design goal is "fully + local, lightweight, no server, no GPU". +- =provider.py= must remain importable outside Hermes: the + =agent.memory_provider= import has a local stub fallback used by the + standalone tests. Keep that pattern intact when editing. +- The embedding model name and dimension live in =src/retinue/memory.py= + as the =MODEL= constant. Changing the model changes the vector + dimension, which changes the =vec0= virtual table schema — existing + databases are not migrated automatically. + +* How to: common tasks +:PROPERTIES: +:ID: e5e31989-cfd9-4152-961c-217b5838115f +:END: + +** Run the standalone tests + +#+begin_src sh +cd /path/to/retinue +"Tools/Run tests" +#+end_src + +Uses =.venv/= and =.hf-cache/= inside the project. If the venv is +missing: + +#+begin_src sh +python3 -m venv .venv +.venv/bin/python -m pip install sqlite-vec model2vec +#+end_src + +** Deploy a change into Hermes + +#+begin_src sh +cd /path/to/retinue +bash "Install Retinue" +hermes memory status # verify +#+end_src + +The installer is idempotent: it wipes everything under +=$HERMES_HOME/plugins/retinue/= except =.venv= / =.hf-cache=, copies the +whitelist =(plugin.yaml __init__.py src)=, installs missing Python deps +into the interpreter Hermes actually runs, pre-downloads the embedding +model into =.hf-cache/= (no first-use stall), sets =memory.provider: +retinue=, and validates the plugin imports. + +To target a non-default Hermes home: + +#+begin_src sh +HERMES_HOME="$HOME/.hermes/profiles/coder" bash "Install Retinue" +#+end_src + +*Restart requirement:* Hermes loads plugins at startup. After every +installer run, any running Hermes session keeps using the previous +copy until restarted. + +** Verify end-to-end inside Hermes + +#+begin_src sh +hermes chat -q "Remember that my default shell is fish." +hermes chat -q "What shell do I use?" +#+end_src + +** Inspect the memory store from the command line + +#+begin_src sh +cd /path/to/retinue +"Tools/Show Retinue memory contents" # dump all memories +"Tools/Show Retinue memory contents" --search "deployment" # semantic search +"Tools/Show Retinue memory contents" --home ~/.hermes/profiles/coder # other Hermes home +"Tools/Show Retinue memory contents" --json | jq . # machine-readable output +#+end_src + +=Show Retinue memory contents= is a debugging helper implemented in +=src/retinue/cli.py=. +It targets =$HERMES_HOME/retinue_memory.db= (auto-detecting a single +profile database under =~/.hermes/profiles/=), with =--db= / =--home= +overrides. Plain listing never loads the embedding model, so it starts +instantly and works offline; only =--search= loads it. It is a +development tool and is intentionally not deployed by the installer. + +** Update the public documentation site + +#+begin_src sh +cd /path/to/retinue +bash "Tools/Update web site" +#+end_src + +This batch-exports every =Documentation/**/*.org= to HTML with +=emacs --batch -l ~/.emacs --visit=... --funcall=org-html-export-to-html= +and rsyncs =Documentation/= to +=n0@www3.svjatoslav.eu:/mnt/big/projects/retinue/= over SSH port 10006. +The script re-execs itself inside a gnome-terminal unless invoked with +a =T= argument; run it with =bash= from a terminal to skip that wrapper. +Never commit the generated =*.html= — they are gitignored. + +* Tooling notes +:PROPERTIES: +:ID: 6f0702f2-2957-4083-a602-caa709ed046e +:END: + +- *Python on Debian 13 is PEP 668.* System pip is blocked. The installer + deliberately installs deps into the Python that runs Hermes + (=PYTHON= env override, =$HERMES_HOME/hermes-agent/venv/bin/python3=, + or the =hermes= launcher shebang), and falls back to + =--break-system-packages= only inside that interpreter. The standalone + tests use the project =.venv=. Do not pip-install into the system + Python from this project. +- *Filenames with spaces.* Everything under =Tools/= contains spaces — + quote paths in shell commands. +- *The =Tools/Open with IntelliJ IDEA= helper* opens this tree as an + IDEA project (=.idea/= is gitignored). +- *Git remote.* Upstream is the user's own git server: + =https://www3.svjatoslav.eu/git/retinue.git= (browse at + =https://www2.svjatoslav.eu/gitweb/?p=retinue.git;a=summary=). + +* Pitfalls +:PROPERTIES: +:ID: ba35d4cc-fb67-4af0-8b73-02c2b6e13cef +:END: + +- *Pitfall: editing the deployed plugin copy.* Changes made in + =~/.hermes/plugins/retinue/= are silently destroyed by the next + installer run. Always edit this workspace tree. +- *Pitfall: forgetting the restart.* The installer finishes green but + Hermes still behaves like the old version — the plugin is loaded at + process start. Restart the Hermes session before concluding a change + did not work. +- *Pitfall: first embedding call is slow (only if the model cache is + missing).* The =minishlab/potion-multilingual-128M= snapshot downloads + ~540 MB on first use into the Hugging Face cache (the repo's unused + ~512 MB ONNX export is skipped — =memory.py= resolves the snapshot + with =snapshot_download(..., ignore_patterns=["onnx/*"])= and loads + from the returned local path, which also prevents re-fetching the + skipped files on every load). The installer pre-downloads it into + =$HERMES_HOME/plugins/retinue/.hf-cache/=, so a normal install never + hits this. If you see a hang on first use, the cache was deleted or + the installer's pre-download step failed (offline install) — re-run + =Install Retinue= with network access. The standalone tests use the + project =.hf-cache/= instead. +- *Pitfall: running tests without the venv.* "Tools/Run tests" refuses + with instructions; do not work around it with system Python (see PEP + 668 note above). +- *Pitfall: model dimension change.* Switching =MODEL= in + =src/retinue/memory.py= changes =get_dim()=, and the existing + =retinue_memory.db= virtual table keeps the old dimension — searches + will fail or return garbage. Delete or rebuild the database when + changing the model. +- *Pitfall: Emacs batch truncation.* When validating .org exports with + Emacs batch mode, never let the script save the buffer — batch + find-file + save has truncated org files on this host. Write with + write_file, validate with a separate read-only batch export, delete + the generated .html afterwards. diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/COPYING @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/Documentation/Development/index.org b/Documentation/Development/index.org new file mode 100644 index 0000000..09863e3 --- /dev/null +++ b/Documentation/Development/index.org @@ -0,0 +1,406 @@ +#+SETUPFILE: ~/.emacs.d/org-styles/html/darksun.theme +#+TITLE: Development - Retinue +#+LANGUAGE: en +#+LATEX_HEADER: \usepackage[margin=1.0in]{geometry} +#+LATEX_HEADER: \usepackage{parskip} +#+LATEX_HEADER: \usepackage[none]{hyphenat} + +#+OPTIONS: H:20 num:20 +#+OPTIONS: author:nil + +[[file:../index.org][Back to main documentation]] + +* Architecture +:PROPERTIES: +:CUSTOM_ID: architecture +:ID: ad252f6e-8152-4dd4-95f6-7b52f773e740 +:END: + +Retinue uses: + +- =model2vec= :: Static embedding model (=minishlab/potion-multilingual-128M=, + ~540 MB download on first use, then offline; the repo's unused ~512 MB + ONNX export is skipped). Understands meaning across languages. +- =sqlite-vec= :: SQLite extension for vector similarity search. No + server, no daemon, no port. +- Unified store :: All memories share one table. Each memory has a + content string, a timestamp, and an auto-generated embedding. + +** Database layout +:PROPERTIES: +:CUSTOM_ID: database-layout +:ID: 2e9bc319-c314-4a48-8fdf-0c6bac11f344 +:END: + +Retinue stores all data in a single SQLite virtual table under the active +Hermes home: + +#+begin_src text +$HERMES_HOME/retinue_memory.db +#+end_src + +*"memories" virtual table:* + +| Column | Type | Meaning | +|-------------+------------------+----------------------------------------------------------| +| =rowid= | INTEGER | Auto-increment primary key. Stable ID used for deletion. | +| =embedding= | float[MODEL_DIM] | The model2vec embedding vector for =content=. | +| =content= | TEXT | The full text of the memory/fact. | +| =created= | TEXT | ISO 8601 UTC timestamp when the row was inserted. | + +=MODEL_DIM= depends on the model (hardcoded to +=minishlab/potion-multilingual-128M=). The table is created with +=sqlite-vec= and supports cosine-similarity search. + +Why a /virtual/ table? In SQLite, a virtual table is created with +=CREATE VIRTUAL TABLE ... USING vec0(...)=. The =vec0= module from +=sqlite-vec= provides the storage and indexing logic for the +=embedding= column, which enables the =embedding MATCH ?= vector +similarity query. The table is still persisted in the same SQLite +=.db= file; “virtual” only means SQLite delegates the table engine to +the =sqlite-vec= extension. + +* Project structure +:PROPERTIES: +:CUSTOM_ID: project-structure +:ID: 6d9ea612-7ecf-4eef-96e0-ab2e9b744a2e +:END: + +#+begin_src +Retinue/ +├── Documentation/ +│ └── index.org # This file +├── Install Retinue # Deploy into the active Hermes plugin directory +├── plugin.yaml # Hermes plugin metadata +├── __init__.py # Plugin entry point: register(ctx) +├── src/ # Plugin source package +│ └── retinue/ +│ ├── __init__.py # Package init +│ ├── memory.py # Memory engine: embeddings + sqlite-vec +│ ├── provider.py # MemoryProvider subclass +│ └── cli.py # Debug CLI behind "Show Retinue memory contents" +├── Tools/ # Development helpers (desktop scripts) +│ ├── Run tests # Standalone smoke test runner +│ ├── Show Retinue memory contents # Debug CLI: dump/search all memories +│ ├── Import Holographic memories # Migrate facts from Holographic memory_store.db +│ └── Update web site # Export Documentation/*.org to HTML + publish +└── $HERMES_HOME/retinue_memory.db # Created at runtime: SQLite + sqlite-vec +#+end_src + +* What the installer does +:PROPERTIES: +:CUSTOM_ID: what-the-installer-does +:ID: 9f6b3951-e8f0-4643-bd54-b35bdf528d6e +:END: + +=Install Retinue= is the deployment bridge between the Retinue workspace and the +active Hermes home. It is intended for iterative development: edit the +source tree, run the script, and test inside Hermes immediately. + +The script is idempotent, so you can run it after every code change. It +preserves runtime artifacts (the database, virtual environments, and caches) +while replacing the plugin source code. + +What it does, in order: + +1. Detects the active Hermes home from the =HERMES_HOME= environment variable, + or falls back to =$HOME/.hermes=. +2. Creates =$HERMES_HOME/plugins/retinue/= and its =.hf-cache/= subdirectory + if they do not exist. +3. Removes every previously deployed item in the plugin directory except the + =PRESERVE= list (=.venv=, =.hf-cache=), so stale code cannot linger. +4. Copies the whitelist =(plugin.yaml __init__.py src)= from the workspace + into the plugin directory with =cp=. +5. Resolves the Python interpreter that Hermes actually runs under (via the + =PYTHON= environment variable, common venv layouts, or the =hermes= + launcher shebang) and installs any missing =sqlite-vec= / =model2vec= + dependencies into it with =pip= (falling back to + =--break-system-packages= inside that interpreter on PEP 668 systems). +6. Pre-downloads the embedding model into =.hf-cache/= by calling + =retinue.memory.get_model()= with the Hermes Python — the same code path + the plugin uses at runtime. The snapshot download skips the repo's + unused ~512 MB ONNX export (=ignore_patterns=["onnx/*"]=) and loads from + the returned local path, so the cache is ~540 MB and the first Hermes + session does not stall. Failure here is non-fatal: a warning is printed + and the model simply downloads on first use instead. +7. Sets =memory.provider: retinue= with =hermes config set= if the CLI is + available. +8. Runs a lightweight import check to confirm the plugin loads. + +To target a specific Hermes profile or home directory: + +#+begin_src sh +HERMES_HOME="$HOME/.hermes/profiles/coder" bash "Install Retinue" +#+end_src + +* Tools exposed to Hermes +:PROPERTIES: +:CUSTOM_ID: tools-exposed-to-hermes +:ID: 04cf7ccf-e71a-4f91-a023-3dbf87d5dbf5 +:END: + +When Retinue is the active memory provider, Hermes can call these tools +natively. The exact names and schemas are defined in +=get_tool_schemas()= and dispatched through =handle_tool_call()=. + +** =retinue_memory_add= +:PROPERTIES: +:CUSTOM_ID: retinue-memory-add +:ID: 69dbc374-8c72-4c8c-a46e-7617bc0c2b2d +:END: + +Store a memory with content. + +#+begin_src json +{ + "name": "retinue_memory_add", + "description": "Store a semantic memory in Retinue.", + "parameters": { + "type": "object", + "properties": { + "content": {"type": "string", "description": "The memory content"} + }, + "required": ["content"] + } +} +#+end_src + +Read more: [[#memory-save][How memory save works]]. + +** =retinue_memory_search= +:PROPERTIES: +:CUSTOM_ID: retinue-memory-search +:ID: 8c479c9b-141e-4a12-9ee0-8f15e398c0c1 +:END: + +Semantic search across all memories. + +#+begin_src json +{ + "name": "retinue_memory_search", + "description": "Search Retinue memories by semantic similarity.", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Natural-language search query"}, + "limit": {"type": "integer", "default": 10} + }, + "required": ["query"] + } +} +#+end_src + +Read more: [[#memory-retrieval][How memory retrieval works]]. + +** =retinue_memory_delete= +:PROPERTIES: +:CUSTOM_ID: retinue-memory-delete +:ID: 4d3fb3e3-6dc3-4184-9c23-dfd0714130db +:END: + +Delete a memory by ID. + +** =retinue_memory_stats= +:PROPERTIES: +:CUSTOM_ID: retinue-memory-stats +:ID: 7d56b6c5-c889-43a3-980e-6b8ed8277ec1 +:END: + +Returns facts about the store itself — not the memories. Takes no +parameters; the LLM calls it with an empty object: + +: retinue_memory_stats({}) + +and gets back a single object: =total= (number of stored memories), +=latest_id= (the rowid assigned to the most recent memory — the next +memory gets =latest_id + 1=, and ids up to it are valid delete +candidates even after deletions), =db_path= (which database file is in +use), =model= (embedding model name), and =dim= (embedding vector +dimension): + +: {"success": true, "result": {"total": 2418, "latest_id": 2675, +: "db_path": "/home/john/.hermes/retinue_memory.db", +: "model": "minishlab/potion-multilingual-128M", "dim": 256}} + +* How memory save works +:PROPERTIES: +:CUSTOM_ID: memory-save +:ID: 6325a3ac-8629-461e-aa5e-62408ba73bde +:END: + +1) LLM decides something is worth remembering it issues the call: + + : retinue_memory_add({"content": "ProjectX is hosted on our primary server"}) + + Alternative trigger: LLM instead calls Hermes' built-in =memory= tool. + Hermes then calls Retinue's =on_memory_write= hook, which feeds the + same text into the same chain at step 2. Both triggers produce one + identical memory in the same single store — one memory system, two + ways to trigger a save. + +2) Hermes routes it to =handle_tool_call("retinue_memory_add", args)=. + +3) Text is embedded with model2vec into a float32 vector packed as bytes. + On the first call in a process, =memory.py= resolves the model snapshot + with =snapshot_download(MODEL, ignore_patterns=["onnx/*"])= (skipping the + repo's unused ~512 MB ONNX export) and loads =StaticModel= from the + returned local path — loading from a local path is what prevents + re-fetching the skipped files on every load; + +4) Resulting vector with original text is written into database + : INSERT INTO memories (embedding, content, created) + +5) JSON string is then returned back to the LLM: + : {"success": true, "result": {"id": 1, "content": "ProjectX is hosted on our primary server"}} + +* How memory retrieval works +:PROPERTIES: +:CUSTOM_ID: memory-retrieval +:ID: e30a9286-6083-48ed-a861-61c72a5737b4 +:END: + +Retrieval happens through exactly one concrete path: the +=retinue_memory_search= tool. The =prefetch= hook is a no-op, so nothing +is injected automatically — every retrieval is an explicit tool call +visible in the conversation transcript. + +The step-by-step mechanics: + +1. *The LLM is notified the tool exists.* At session start Hermes calls + =get_tool_schemas()= and places all four =retinue_memory_*= schemas + into the LLM's tool list next to the built-in tools. The + =retinue_memory_search= schema declares two parameters: =query= + (required string) and =limit= (optional integer, default 50). + Separately, the text of =src/retinue/system_prompt_block.txt= is + injected into the system prompt, instructing the LLM to run + =retinue_memory_search= before acting on a task. + +2. *The LLM invokes it with a natural-language query.* Example call: + =retinue_memory_search({"query": "coworker", "limit": 2})=. + +3. *Retinue embeds the query and runs a vector query.* The query text is + embedded with the same model2vec model used at write time, then + =MemoryStore.search()= executes a =sqlite-vec= KNN query: + =WHERE embedding MATCH ? AND k = = against the =memories= + virtual table. + +4. *Result count is limited by count only.* The =k= parameter is a + hard top-N cap: sqlite-vec returns at most =limit= nearest rows, + *with no similarity threshold*. Even a nonsense query (="zzzzqqq + nonexistent"=) returns exactly =limit= rows. Filtering weak matches + is left to the LLM reading the scores. + +5. *Results are sorted best-first.* sqlite-vec orders KNN results by + ascending distance, so the most similar memory is always + =results[0]=. Ordering is deterministic; ties can occur (two + different memories scored 0.458 on the same query). + +6. *The tool returns a JSON string* — the result of + =json.dumps({"success": True, "result": [...]})=. Each element has + four fields: =id= (the rowid, for =retinue_memory_delete=), =content= + (full stored text), =created= (ISO 8601 UTC timestamp), and + =similarity=. The =similarity= is computed in =provider.py= from the + raw sqlite-vec distance as =1 / (1 + distance)= — higher is better, + 1.0 means identical vectors. (Raw distance in the database is + unbounded L2 on the model's non-normalized embeddings, so typical + real-world matches land in the 0.4–0.6 range; treat the score as a + relative ranking signal, not an absolute confidence.) + + #+begin_src + {"success": true, "result": [ + {"id": 2, "content": "Coworker John reads Spanish", + "created": "2026-07-17T20:33:43.081952+00:00", "similarity": 0.4626}, + {"id": 1, "content": "ProjectX is hosted on our serverY", + "created": "2026-07-17T20:33:40.987271+00:00", "similarity": 0.4357} + ]} + #+end_src + + (Captured verbatim from =handle_tool_call("retinue_memory_search", + {"query": "coworker", "limit": 2})= against a store holding exactly + those two memories; similarity values shortened for print.) + + +7. *The LLM reads the JSON and decides.* Weak-looking results (all + scores near the noise floor) mean the store has nothing relevant; + the LLM proceeds without memory context. + +* Development and testing workflow +:PROPERTIES: +:CUSTOM_ID: development-and-testing-workflow +:ID: 246ce71b-6841-49fb-a330-2bb83fc275d3 +:END: + +Retinue is developed in its own source tree and deployed into the active +Hermes home with =Install Retinue=. This makes iteration fast and safe: the +runtime =data/= directory lives in Hermes, while the source code lives in +the workspace. + +** Iterative development loop +:PROPERTIES: +:CUSTOM_ID: iterative-development-loop +:ID: b5b488be-6961-4b2a-ae81-f11d37b73c0b +:END: + +1. Edit source files in the Retinue workspace. +2. Run the installer to copy the latest code into Hermes: + + #+begin_src sh + cd /path/to/retinue + bash "Install Retinue" + #+end_src + +3. Verify the plugin loads: + + #+begin_src sh + hermes memory status + #+end_src + +4. Test through a Hermes session: + + #+begin_src sh + hermes chat -q "Remember that my default shell is fish." + hermes chat -q "What shell do I use?" + #+end_src + +5. Repeat from step 1. + +** Standalone testing +:PROPERTIES: +:CUSTOM_ID: standalone-testing +:ID: 24aa4d55-e730-451c-aa27-9b9f4e5959c7 +:END: + +You can also test Retinue without installing it into Hermes. The project +includes a standalone test runner: + +#+begin_src sh +cd /path/to/retinue +"Tools/Run tests" +#+end_src + +This uses a project-local =.venv= and caches downloaded models in +=.hf-cache=, both of which are excluded from deployment by the installer. + +** Inspecting the memory store +:PROPERTIES: +:CUSTOM_ID: inspecting-the-memory-store +:ID: 3b7f2c8e-5a1d-4e9f-b6c0-9d2e4f7a1b3c +:END: + +For debugging, the =Show Retinue memory contents= script dumps every stored memory to +the terminal without going through Hermes: + +#+begin_src sh +cd /path/to/retinue +"Tools/Show Retinue memory contents" # dump all memories +"Tools/Show Retinue memory contents" --search "deployment" # semantic search +"Tools/Show Retinue memory contents" --home ~/.hermes/profiles/coder # other Hermes home +"Tools/Show Retinue memory contents" --json # machine-readable output +#+end_src + +It resolves the database from =--db=, =--home=, =$HERMES_HOME=, or by +auto-detecting a single profile database under =~/.hermes/profiles/=. +Plain listing never loads the embedding model, so it works offline and +starts instantly; only =--search= loads it (~540 MB into =.hf-cache= on +first use). The implementation lives in =src/retinue/cli.py= and is not +deployed by the installer — it is a development-only tool. diff --git a/Documentation/index.org b/Documentation/index.org new file mode 100644 index 0000000..8c84af2 --- /dev/null +++ b/Documentation/index.org @@ -0,0 +1,194 @@ +:PROPERTIES: +:ID: ee70e32d-0ead-4236-bbc2-9dc321dc6a92 +:END: +#+SETUPFILE: ~/.emacs.d/org-styles/html/darksun.theme +#+TITLE: Retinue — Memory provider plugin for Hermes Agent +#+LANGUAGE: en +#+LATEX_HEADER: \usepackage[margin=1.0in]{geometry} +#+LATEX_HEADER: \usepackage{parskip} +#+LATEX_HEADER: \usepackage[none]{hyphenat} + +#+OPTIONS: H:20 num:20 +#+OPTIONS: author:nil + +* Overview +:PROPERTIES: +:CUSTOM_ID: overview +:ID: a22b19b1-bc50-4368-a0c0-ae349f732df1 +:END: + +[[file:logo.png]] + +*Retinue* is a [[https://hermes-agent.nousresearch.com/][Hermes Agent]] memory provider plugin that adds +persistent, semantic memory to every Hermes session. Once installed +and configured, it becomes Hermes' external memory backend. + +Note: While Hermes already has multiple built-in memory systems, I did +not find a solution that: +- Runs fully locally, *and* +- Offers semantic memory retrieval (find memories by meaning, not by the + exact keyword used), *and* +- Is lightweight (runs fast using CPU-only compute, with no need for a + GPU) + +It is built on =model2vec= static embeddings and =sqlite-vec=, so it +runs offline, requires no server, and stores everything in a single +SQLite database. + + +For technical details, read: +- [[file:Development/index.org::#memory-save][How memory save works]]. +- [[file:Development/index.org::#memory-retrieval][How memory retrieval works]]. + +* Installation +:PROPERTIES: +:CUSTOM_ID: installation +:ID: bc83f42b-b126-4b48-b365-762d27618651 +:END: + +Prerequisites: Hermes Agent must be installed and the =hermes= CLI must be on +your =PATH=. + +Run the bundled installer from the Retinue source tree: + +#+begin_src sh +cd /path/to/retinue +bash "Install Retinue" +#+end_src + +The installer performs four steps: + +1. Detects the active Hermes home from the =HERMES_HOME= environment variable + or falls back to =$HOME/.hermes=. +2. Resolves the Python interpreter that Hermes uses and installs =sqlite-vec= + and =model2vec= there if they are missing. +3. Copies the Retinue source into =$HERMES_HOME/plugins/retinue/=, preserving + any existing runtime data. +4. Pre-downloads the embedding model + (=minishlab/potion-multilingual-128M=, ~540 MB — the repo's unused + ONNX export is skipped) into + =$HERMES_HOME/plugins/retinue/.hf-cache/= so the first Hermes session + does not stall on the download. The cache is preserved across + re-installs, so this step is a no-op after the first successful run. +5. Sets =memory.provider: retinue=. + +Verify the plugin is active: + +#+begin_src sh +hermes memory status +#+end_src + +*Important:* If Hermes is already running, restart it. Hermes loads plugins +at startup, so an active session will not see the newly installed Retinue plugin +until it is restarted. + +For technical details, read: [[file:Development/index.org::#what-the-installer-does][What the installer does.]] + +** Migrating from Holographic memory +:PROPERTIES: +:CUSTOM_ID: migrating-from-holographic +:ID: a1b2c3d4-e5f6-7890-abcd-ef1234567890 +:END: + +If you previously used the Holographic memory provider plugin (the +=fact_store= tool with =memory_store.db=), you can import your old facts +into Retinue with the bundled migration tool. + +Run the importer from the Retinue source tree: + +#+begin_src sh +cd /path/to/retinue +"Tools/Import Holographic memories" --source /path/to/memory_store.db +#+end_src + +The importer performs three steps: + +1. Reads every fact from the Holographic =memory_store.db=. +2. Embeds each fact's content with the Retinue embedding model. +3. Writes new memories into the active Retinue database, skipping exact + duplicates. + +To preview what would be imported without writing anything: + +#+begin_src sh +"Tools/Import Holographic memories" --source /path/to/memory_store.db --dry-run +#+end_src + +* Updating +:PROPERTIES: +:CUSTOM_ID: updating +:ID: 3c4e5f6a-7b8c-9d0e-1a2b-3c4d5e6f7a8b +:END: + +To install an update to Retinue, run the same installer again from the +latest source tree: + +#+begin_src sh +cd /path/to/retinue +bash "Install Retinue" +#+end_src + +The installer is idempotent: it will replace the deployed plugin code +with the current source, install any missing or updated Python +dependencies, and leave the existing memory database untouched. + +To verify the update after re-running the installer: + +#+begin_src sh +hermes memory status +#+end_src + +*Important:* Restart any running Hermes sessions. Hermes loads plugins at +startup, so a running session will continue to use the previous version of +Retinue until it is restarted. + +For technical details, read: [[file:Development/index.org::#what-the-installer-does][What the installer does.]] + +* Quick start +:PROPERTIES: +:CUSTOM_ID: quick-start +:ID: e1341a37-be8c-4c3b-810e-4e34a222406d +:END: + +After installing the plugin, try storing and recalling a memory through a +Hermes session: + +1. Store a fact: + + #+begin_src text + Remember that ProjectX is hosted on our primary server. + #+end_src + + Hermes will call =retinue_memory_add= with the content. + +2. Later, ask a related question: + + #+begin_src text + Where is ProjectX hosted? + #+end_src + + Hermes will call =retinue_memory_search= and retrieve the relevant memory. + +* Source code +:PROPERTIES: +:CUSTOM_ID: development +:END: + +*This program is free software: released under Creative Commons Zero +(CC0) license* + +*Program author:* +- Svjatoslav Agejenko +- Homepage: https://svjatoslav.eu +- Email: mailto://svjatoslav@svjatoslav.eu +- See also: [[https://www.svjatoslav.eu/projects/][Other software projects hosted at svjatoslav.eu]] + +*Getting the source code:* +- [[https://www2.svjatoslav.eu/gitweb/?p=retinue.git;a=snapshot;h=HEAD;sf=tgz][Download latest source code snapshot in TAR GZ format]] +- [[https://www2.svjatoslav.eu/gitweb/?p=retinue.git;a=summary][Browse Git repository online]] +- Clone Git repository using command: + : git clone https://www3.svjatoslav.eu/git/retinue.git + + +For detailed architecture, integration with Hermes Agent, and the +project development workflow, see the dedicated [[file:Development/][Development]] page. diff --git a/Documentation/logo.png b/Documentation/logo.png new file mode 100644 index 0000000..7bc3f13 Binary files /dev/null and b/Documentation/logo.png differ diff --git a/Install Retinue b/Install Retinue new file mode 100755 index 0000000..395cf51 --- /dev/null +++ b/Install Retinue @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# Install Retinue - Deploy Retinue into the active Hermes home as a memory +# provider plugin. +# +# Usage: +# bash "Install Retinue" +# HERMES_HOME="$HOME/.hermes/profiles/coder" bash "Install Retinue" +# +# This script is idempotent: run it again after every code change to +# re-deploy the latest version into Hermes. It never overwrites the +# runtime data/ directory. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_DIR="$SCRIPT_DIR" + +HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}" +PLUGIN_DIR="$HERMES_HOME/plugins/retinue" + +# Resolve the Python interpreter that actually runs Hermes. Plugin deps must +# live in that environment, not the user's system Python. +resolve_hermes_python() { + # Allow explicit override. + if [[ -n "${PYTHON:-}" ]]; then + command -v "$PYTHON" >/dev/null 2>&1 && { echo "$PYTHON"; return; } + fi + + # Common Hermes git install layout. + local venv_python="$HERMES_HOME/hermes-agent/venv/bin/python3" + if [[ -x "$venv_python" ]]; then + echo "$venv_python" + return + fi + + # Derive from the hermes launcher shebang. + local hermes_bin + hermes_bin="$(command -v hermes 2>/dev/null || true)" + if [[ -n "$hermes_bin" && -r "$hermes_bin" ]]; then + local shebang + shebang="$(head -n1 "$hermes_bin" 2>/dev/null || true)" + if [[ "$shebang" == \#\!* ]]; then + shebang="${shebang#\#!}" + shebang="$(echo -n "$shebang" | sed 's/^[[:space:]]*//')" + if [[ -x "$shebang" ]]; then + echo "$shebang" + return + fi + fi + fi + + # Fallback. + echo "python3" +} + +PYTHON="$(resolve_hermes_python)" + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + echo "Usage: bash \"Install Retinue\"" + echo "" + echo "Environment:" + echo " HERMES_HOME Target Hermes home (default: $HOME/.hermes)" + echo " PYTHON Python interpreter to use (default: python3)" + exit 0 + ;; + *) + echo "Unknown argument: $1" + echo "Run 'bash \"Install Retinue\" --help' for usage." + exit 1 + ;; + esac +done + +echo "=== Retinue installer ===" +echo "Source: $SOURCE_DIR" +echo "Target: $PLUGIN_DIR" + +if [[ "$SOURCE_DIR" == "$PLUGIN_DIR" ]]; then + echo "ERROR: source and target directories are the same." + echo " Run \"Install Retinue\" from the Retinue source tree, not the plugin directory." + exit 1 +fi + +mkdir -p "$PLUGIN_DIR" + +# Deploy only the files/directories required by the plugin at runtime. +# Everything else (tests, README, this installer, IDE metadata, etc.) is left behind. +WHITELIST=(plugin.yaml __init__.py src) +PRESERVE=(.venv .hf-cache) + +# Create the Hugging Face cache directory so the model pre-download step +# has somewhere to write even on a completely fresh install. +mkdir -p "$PLUGIN_DIR/.hf-cache" + +echo "Deploying plugin files..." + +# Remove old deployed items, but preserve runtime directories. +find "$PLUGIN_DIR" -mindepth 1 -maxdepth 1 -print0 | while IFS= read -r -d '' entry; do + name="$(basename "$entry")" + for keep in "${PRESERVE[@]}"; do + if [[ "$name" == "$keep" ]]; then + echo " Preserving: $entry" + continue 2 + fi + done + echo " Removing: $entry" + rm -rf "$entry" +done + +# Copy the whitelisted items from the source tree. +for item in "${WHITELIST[@]}"; do + if [[ -e "$SOURCE_DIR/$item" ]]; then + cp -r "$SOURCE_DIR/$item" "$PLUGIN_DIR/" + fi +done + +# Ensure runtime data directory exists +# (Retinue stores runtime data under HERMES_HOME/retinue, not in the plugin directory.) +# mkdir -p "$PLUGIN_DIR/data" + +# Ensure Python dependencies are available in the target interpreter. +# Hermes itself runs with this Python, so the deps must be importable there. +ensure_python_deps() { + local missing="" + if ! "$PYTHON" -c "import sqlite_vec" 2>/dev/null; then + missing="$missing sqlite-vec" + fi + if ! "$PYTHON" -c "import model2vec" 2>/dev/null; then + missing="$missing model2vec" + fi + if [[ -z "$missing" ]]; then + return 0 + fi + + echo "Missing Python dependencies:$missing" + echo "Installing with $PYTHON -m pip ..." + + if "$PYTHON" -m pip install $missing 2>/dev/null; then + echo "Dependencies installed successfully." + return 0 + fi + + if "$PYTHON" -m pip install --break-system-packages $missing 2>/dev/null; then + echo "Dependencies installed successfully (with --break-system-packages)." + return 0 + fi + + echo "ERROR: Failed to install dependencies." + echo " Install them manually:" + echo " $PYTHON -m pip install --break-system-packages sqlite-vec model2vec" + return 1 +} + +ensure_python_deps + +# Pre-download the embedding model into the plugin-local cache so the first +# Hermes session does not hang on a ~500 MB download. +# HF_HOME is preserved across installer runs (PRESERVE list above), so this +# is a no-op on re-installs after the first successful download. +preload_embedding_model() { + echo "Pre-downloading embedding model (minishlab/potion-multilingual-128M)..." + export HF_HOME="$PLUGIN_DIR/.hf-cache" + if "$PYTHON" -c " +import sys +sys.path.insert(0, '$PLUGIN_DIR/src') +from retinue.memory import get_model +get_model() +print(' Embedding model ready.') +" 2>/dev/null; then + echo " Model cached at $PLUGIN_DIR/.hf-cache" + else + echo " Warning: could not pre-download the embedding model." + echo " It will download on first use inside Hermes." + echo " (Check network / Hugging Face connectivity.)" + fi +} + +preload_embedding_model + +# Set Hermes memory provider if the CLI is available. +# This may fail in read-only sandbox configs; the user can set it manually. +if command -v hermes &>/dev/null; then + echo "Setting memory.provider to retinue..." + if hermes config set memory.provider retinue 2>/dev/null; then + echo " memory.provider set to retinue." + else + echo " Warning: could not set memory.provider automatically." + echo " Set it manually in $HERMES_HOME/config.yaml:" + echo " memory:" + echo " provider: retinue" + fi +else + echo "Warning: hermes CLI not found. Set memory.provider: retinue manually in:" + echo " $HERMES_HOME/config.yaml" +fi + +# Validate that the plugin imports. +echo "Validating plugin import with: $PYTHON" +if "$PYTHON" -c " +import sys +sys.path.insert(0, '$PLUGIN_DIR/src') +from retinue.provider import RetinueMemoryProvider +p = RetinueMemoryProvider() +print('name:', p.name) +print('available:', p.is_available()) +print('schemas:', [s['name'] for s in p.get_tool_schemas()]) +" 2>/dev/null; then + echo "Plugin validation passed." +else + echo "Plugin validation failed. Dependencies may be missing." + echo "Install them with:" + echo " $PYTHON -m pip install sqlite-vec model2vec" + echo "The embedding model (~500 MB) will download on first use." + exit 1 +fi + +echo "" +echo "=== Installation complete ===" +echo "" +echo "The embedding model is pre-downloaded and ready — no first-use delay." +echo "" +echo "Next steps:" +if command -v hermes &>/dev/null; then + echo " hermes memory setup" + echo " hermes memory status" + echo " hermes chat -q 'Remember something...'" +else + echo " Install hermes CLI, then:" + echo " hermes memory setup" + echo " hermes memory status" +fi diff --git a/Tools/Import Holographic memories b/Tools/Import Holographic memories new file mode 100755 index 0000000..eba9e61 --- /dev/null +++ b/Tools/Import Holographic memories @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Import Holographic memories - migrate facts from the old Holographic +# memory provider (memory_store.db) into Retinue's semantic memory store. +# +# Uses the project-local venv (same as "Run tests"), so dependencies and +# the Hugging Face cache live inside the project tree. +# +# Usage: +# "Tools/Import Holographic memories" [--source PATH] [--db PATH | --home PATH] +# [--dry-run] [--json] [--verbose] +# +# Examples: +# "Tools/Import Holographic memories" --source ~/.hermes/memory_store.db +# "Tools/Import Holographic memories" --source "/mnt/backup/.hermes" # directory works too +# "Tools/Import Holographic memories" --dry-run --verbose +# "Tools/Import Holographic memories" --json +# +# The embedding model (~500 MB) downloads on first use into .hf-cache. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +VENV="$PROJECT_ROOT/.venv" + +if [[ ! -d "$VENV" ]]; then + echo "Virtual environment not found at $VENV" + echo "Create it with:" + echo " python3 -m venv $VENV" + echo " $VENV/bin/python -m pip install sqlite-vec model2vec" + exit 1 +fi + +export HF_HOME="${HF_HOME:-$PROJECT_ROOT/.hf-cache}" +export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" + +exec "$VENV/bin/python" -m retinue.import_holographic "$@" diff --git a/Tools/Open with IntelliJ IDEA b/Tools/Open with IntelliJ IDEA new file mode 100755 index 0000000..304bf94 --- /dev/null +++ b/Tools/Open with IntelliJ IDEA @@ -0,0 +1,54 @@ +#!/bin/bash + +# This script launches IntelliJ IDEA with the current project +# directory. The script is designed to be run by double-clicking it in +# the GNOME Nautilus file manager. + +# First, we change the current working directory to the directory of +# the script. + +# "${0%/*}" gives us the path of the script itself, without the +# script's filename. + +# This command basically tells the system "change the current +# directory to the directory containing this script". + +cd "${0%/*}" + +# Then, we move up one directory level. +# The ".." tells the system to go to the parent directory of the current directory. +# This is done because we assume that the project directory is one level up from the script. +cd .. + +# Now, we use the 'setsid' command to start a new session and run +# IntelliJ IDEA in the background. 'setsid' is a UNIX command that +# runs a program in a new session. + +# The command 'idea .' opens IntelliJ IDEA with the current directory +# as the project directory. The '&' at the end is a UNIX command that +# runs the process in the background. The '> /dev/null' part tells +# the system to redirect all output (both stdout and stderr, denoted +# by '&') that would normally go to the terminal to go to /dev/null +# instead, which is a special file that discards all data written to +# it. + +setsid idea . &>/dev/null & + +# The 'disown' command is a shell built-in that removes a shell job +# from the shell's active list. Therefore, the shell will not send a +# SIGHUP to this particular job when the shell session is terminated. + +# '-h' option specifies that if the shell receives a SIGHUP, it also +# doesn't send a SIGHUP to the job. + +# '$!' is a shell special parameter that expands to the process ID of +# the most recent background job. +disown -h $! + + +sleep 2 + +# Finally, we use the 'exit' command to terminate the shell script. +# This command tells the system to close the terminal window after +# IntelliJ IDEA has been opened. +exit diff --git a/Tools/Run tests b/Tools/Run tests new file mode 100755 index 0000000..deae42c --- /dev/null +++ b/Tools/Run tests @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Run tests - Standalone test runner for Retinue, no Hermes installation required. +# This uses the project-local venv and caches models under the project tree +# so everything survives across agent instances. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +VENV="$PROJECT_ROOT/.venv" +TEST="$PROJECT_ROOT/test/test_standalone.py" + +if [[ ! -d "$VENV" ]]; then + echo "Virtual environment not found at $VENV" + echo "Create it with:" + echo " python3 -m venv $VENV" + echo " $VENV/bin/python -m pip install sqlite-vec model2vec" + exit 1 +fi + +export HF_HOME="${HF_HOME:-$PROJECT_ROOT/.hf-cache}" + +exec "$VENV/bin/python" "$TEST" diff --git a/Tools/Show Retinue memory contents b/Tools/Show Retinue memory contents new file mode 100755 index 0000000..9a1a1bc --- /dev/null +++ b/Tools/Show Retinue memory contents @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Show Retinue memory contents - dump all Retinue semantic memories to the +# terminal. Debugging helper. +# Uses the project-local venv (same as "Run tests"), so dependencies and +# the Hugging Face cache live inside the project tree. +# +# Usage: +# "Tools/Show Retinue memory contents" [--db PATH | --home PATH] [--search QUERY] [--limit N] [--json] +# +# Examples: +# "Tools/Show Retinue memory contents" # dump everything +# "Tools/Show Retinue memory contents" --search "deployment" # semantic search +# "Tools/Show Retinue memory contents" --home ~/.hermes/profiles/coder # other Hermes home +# +# Listing works offline and never downloads the embedding model; only +# --search loads it (~500 MB into .hf-cache on first use). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +VENV="$PROJECT_ROOT/.venv" + +if [[ ! -d "$VENV" ]]; then + echo "Virtual environment not found at $VENV" + echo "Create it with:" + echo " python3 -m venv $VENV" + echo " $VENV/bin/python -m pip install sqlite-vec model2vec" + exit 1 +fi + +export HF_HOME="${HF_HOME:-$PROJECT_ROOT/.hf-cache}" +export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" + +exec "$VENV/bin/python" -m retinue.cli "$@" diff --git a/Tools/Update web site b/Tools/Update web site new file mode 100755 index 0000000..76f22ae --- /dev/null +++ b/Tools/Update web site @@ -0,0 +1,80 @@ +#!/bin/bash +cd "${0%/*}"; if [ "$1" != "T" ]; then gnome-terminal -e "'$0' T"; exit; fi; + +cd .. + +# Function to export org to html using emacs in batch mode +export_org_to_html() { + local org_file=$1 + local dir=$(dirname "$org_file") + local base=$(basename "$org_file" .org) + ( + cd "$dir" || return 1 + local html_file="${base}.html" + if [ -f "$html_file" ]; then + rm -f "$html_file" + fi + echo "Exporting: $org_file → $dir/$html_file" + emacs --batch -l ~/.emacs --visit="${base}.org" --funcall=org-html-export-to-html --kill + if [ $? -eq 0 ]; then + echo "✓ Successfully exported $org_file" + else + echo "✗ Failed to export $org_file" + return 1 + fi + ) +} + +export_org_files_to_html() { + echo "🔍 Searching for .org files in Documentation/ ..." + echo "=======================================" + + mapfile -t ORG_FILES < <(find Documentation -type f -name "*.org" | sort) + + if [ ${#ORG_FILES[@]} -eq 0 ]; then + echo "❌ No .org files found!" + return 1 + fi + + echo "Found ${#ORG_FILES[@]} .org file(s):" + printf '%s\n' "${ORG_FILES[@]}" + echo "=======================================" + + SUCCESS_COUNT=0 + FAILED_COUNT=0 + + for org_file in "${ORG_FILES[@]}"; do + export_org_to_html "$org_file" + if [ $? -eq 0 ]; then + ((SUCCESS_COUNT++)) + else + ((FAILED_COUNT++)) + fi + done + + echo "=======================================" + echo "📊 SUMMARY:" + echo " ✓ Successful: $SUCCESS_COUNT" + echo " ✗ Failed: $FAILED_COUNT" + echo " Total: $((SUCCESS_COUNT + FAILED_COUNT))" + echo "" +} + +# Publish Emacs org-mode files into HTML format +export_org_files_to_html + +# Upload assembled documentation to server +SERVER_DIR="n0@www3.svjatoslav.eu:/mnt/big/projects/retinue/" + +echo "📤 Uploading to $SERVER_DIR ..." +rsync -avz --delete -e 'ssh -p 10006' Documentation/ "$SERVER_DIR" + +if [ $? -eq 0 ]; then + echo "✓ Upload completed successfully!" +else + echo "✗ Upload failed!" +fi + +echo "" +echo "Press ENTER to close this window." +read diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..fb5c013 --- /dev/null +++ b/__init__.py @@ -0,0 +1,20 @@ +"""Retinue Hermes memory provider plugin entry point. + +When Hermes loads the plugin from plugins/memory/retinue/, it imports this +file and calls register(ctx). +""" + +import sys +from pathlib import Path + +# Make src/retinue importable from the plugin directory. +_heresrc = Path(__file__).parent / "src" +if str(_heresrc) not in sys.path: + sys.path.insert(0, str(_heresrc)) + +from retinue.provider import RetinueMemoryProvider + + +def register(ctx) -> None: + """Called by the Hermes memory plugin discovery system.""" + ctx.register_memory_provider(RetinueMemoryProvider()) diff --git a/plugin.yaml b/plugin.yaml new file mode 100644 index 0000000..5c63698 --- /dev/null +++ b/plugin.yaml @@ -0,0 +1,6 @@ +name: retinue +version: 1.1.0 +description: "Semantic memory provider for Hermes Agent using model2vec and sqlite-vec." +hooks: + - sync_turn + - shutdown diff --git a/src/retinue/__init__.py b/src/retinue/__init__.py new file mode 100644 index 0000000..8209387 --- /dev/null +++ b/src/retinue/__init__.py @@ -0,0 +1,12 @@ +"""Retinue package.""" + +from retinue.memory import MemoryStore, embed_text, get_dim, get_model +from retinue.provider import RetinueMemoryProvider + +__all__ = [ + "MemoryStore", + "RetinueMemoryProvider", + "embed_text", + "get_dim", + "get_model", +] diff --git a/src/retinue/cli.py b/src/retinue/cli.py new file mode 100644 index 0000000..8a89355 --- /dev/null +++ b/src/retinue/cli.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Dump all Retinue semantic memories to the terminal. + +Debugging helper. Lists every memory stored in the Retinue database in +human-readable form, optionally filtering by semantic similarity. + +Usage: + python3 -m retinue.cli [--db PATH | --home PATH] [--search QUERY] + [--limit N] [--json] + +The database is located in this order: + 1. --db PATH explicit path to retinue_memory.db + 2. --home PATH explicit Hermes home (looks for retinue_memory.db) + 3. $HERMES_HOME environment variable + 4. Auto-detect: ~/.hermes/profiles/*/retinue_memory.db (if exactly + one exists), else ~/.hermes/retinue_memory.db + +Listing and stats do NOT load the embedding model, so they work offline +and start instantly. Only --search needs the model +(minishlab/potion-multilingual-128M, ~500 MB download on first use). +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +from retinue.memory import MemoryStore + +MODEL_HINT = ( + "The embedding model (minishlab/potion-multilingual-128M, ~500 MB) " + "will be downloaded on first use." +) + + +def _detect_db(args: argparse.Namespace) -> Path: + """Resolve which database file to open, per the documented order.""" + if args.db: + return Path(args.db).expanduser() + if args.home: + return Path(args.home).expanduser() / "retinue_memory.db" + env_home = os.environ.get("HERMES_HOME") + if env_home: + return Path(env_home).expanduser() / "retinue_memory.db" + default_home = Path.home() / ".hermes" + profiles = sorted( + (default_home / "profiles").glob("*/retinue_memory.db") + ) if (default_home / "profiles").is_dir() else [] + if len(profiles) == 1: + return profiles[0] + return default_home / "retinue_memory.db" + + +def _print_row(memory_id: int, content: str, created: str, + similarity: float | None = None) -> None: + header = f"[{memory_id}] {created}" + if similarity is not None: + header += f" (similarity {similarity:.3f})" + print(header) + for line in content.splitlines() or [""]: + print(f" {line}") + print() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="show-retinue-memory-contents", + description="Dump all Retinue memories to the terminal (debugging tool).", + ) + parser.add_argument("--db", metavar="PATH", + help="path to retinue_memory.db") + parser.add_argument("--home", metavar="PATH", + help="Hermes home directory (default: $HERMES_HOME or ~/.hermes)") + parser.add_argument("--search", metavar="QUERY", + help="semantic search instead of a full dump " + "(loads the embedding model)") + parser.add_argument("--limit", metavar="N", type=int, default=50, + help="max results for --search (default: 50)") + parser.add_argument("--json", action="store_true", + help="emit JSON instead of human-readable text") + args = parser.parse_args(argv) + + db_path = _detect_db(args) + if not db_path.is_file(): + print(f"error: Retinue database not found: {db_path}", file=sys.stderr) + print("Use --db PATH or --home PATH to point at the right location.", + file=sys.stderr) + return 1 + + store = MemoryStore(base_dir=db_path.parent) + try: + if args.search: + try: + rows = store.search(args.search, limit=args.limit) + except RuntimeError as e: + print(f"error: {e}", file=sys.stderr) + print(MODEL_HINT, file=sys.stderr) + return 1 + results = [ + { + "id": row[0], + "content": row[1], + "created": row[2], + "similarity": max(0.0, 1.0 / (1.0 + row[3])), + } + for row in rows + ] + if args.json: + print(json.dumps({ + "db_path": str(db_path), + "query": args.search, + "results": results, + }, indent=2, ensure_ascii=False)) + else: + print(f"Database: {db_path}") + print(f"Query: {args.search!r} — {len(results)} match(es)\n") + for r in results: + _print_row(r["id"], r["content"], r["created"], + r["similarity"]) + else: + rows = store.list() + if args.json: + print(json.dumps({ + "db_path": str(db_path), + "total": len(rows), + "memories": [ + {"id": row[0], "content": row[1], "created": row[2]} + for row in rows + ], + }, indent=2, ensure_ascii=False)) + else: + print(f"Database: {db_path}") + print(f"Memories: {len(rows)}\n") + for row in rows: + _print_row(row[0], row[1], row[2]) + return 0 + finally: + store.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/retinue/import_holographic.py b/src/retinue/import_holographic.py new file mode 100644 index 0000000..744712f --- /dev/null +++ b/src/retinue/import_holographic.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Import memories from a Holographic memory store (memory_store.db) into Retinue. + +Reads facts from the old Holographic MemoryProvider SQLite database and writes +them into Retinue's semantic memory store. This facilitates migration for +users who previously used the Holographic memory plugin and want to switch +to Retinue. + +Usage: + python3 -m retinue.import_holographic [--source PATH] [--db PATH | --home PATH] + [--dry-run] [--json] [--verbose] + +The source database is located in this order: + 1. --source PATH memory_store.db file, or a directory containing it + 2. $HERMES_HOME environment variable (looks for memory_store.db) + 3. ~/.hermes/memory_store.db + +The target Retinue database is located in this order: + 1. --db PATH explicit path to retinue_memory.db + 2. --home PATH explicit Hermes home (looks for retinue_memory.db) + 3. $HERMES_HOME environment variable + 4. Auto-detect: ~/.hermes/profiles/*/retinue_memory.db (if exactly + one exists), else ~/.hermes/retinue_memory.db + +Notes: + - Only the fact content is imported; trust scores, retrieval counts, and + HRR vectors are discarded (Retinue uses semantic embeddings instead). + - Duplicate facts (same content) are skipped. + - The embedding model (minishlab/potion-multilingual-128M, ~500 MB) is + downloaded on first use into .hf-cache. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sqlite3 +import sys +from pathlib import Path + +from retinue.memory import MemoryStore + +MODEL_HINT = ( + "The embedding model (minishlab/potion-multilingual-128M, ~500 MB) " + "will be downloaded on first use." +) + + +def _detect_source(args: argparse.Namespace) -> Path: + """Resolve the Holographic memory_store.db path. + + --source accepts either the database file itself or a directory that + contains it (e.g. a backed-up Hermes home). + """ + candidate: Path + if args.source: + candidate = Path(args.source).expanduser() + else: + env_home = os.environ.get("HERMES_HOME") + candidate = (Path(env_home).expanduser() if env_home + else Path.home() / ".hermes") / "memory_store.db" + if candidate.is_dir() or (not candidate.exists() and candidate.suffix != ".db"): + candidate = candidate / "memory_store.db" + return candidate + + +def _detect_target(args: argparse.Namespace) -> Path: + """Resolve the Retinue retinue_memory.db path.""" + if args.db: + return Path(args.db).expanduser() + if args.home: + return Path(args.home).expanduser() / "retinue_memory.db" + env_home = os.environ.get("HERMES_HOME") + if env_home: + return Path(env_home).expanduser() / "retinue_memory.db" + default_home = Path.home() / ".hermes" + profiles = sorted( + (default_home / "profiles").glob("*/retinue_memory.db") + ) if (default_home / "profiles").is_dir() else [] + if len(profiles) == 1: + return profiles[0] + return default_home / "retinue_memory.db" + + +def _read_holographic_facts(db_path: Path) -> list[dict]: + """Read all facts from the Holographic memory_store.db.""" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute( + """ + SELECT fact_id, content, category, tags, trust_score, + retrieval_count, helpful_count, created_at, updated_at + FROM facts + ORDER BY fact_id + """ + ).fetchall() + return [dict(r) for r in rows] + finally: + conn.close() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="import-holographic-memories", + description="Import facts from a Holographic memory_store.db into Retinue.", + ) + parser.add_argument( + "--source", metavar="PATH", + help="Holographic memory_store.db file, or a directory containing it " + "(default: $HERMES_HOME/memory_store.db, else ~/.hermes/memory_store.db)", + ) + parser.add_argument( + "--db", metavar="PATH", + help="path to Retinue retinue_memory.db", + ) + parser.add_argument( + "--home", metavar="PATH", + help="Hermes home directory (default: $HERMES_HOME or ~/.hermes)", + ) + parser.add_argument( + "--dry-run", action="store_true", + help="show what would be imported without writing anything", + ) + parser.add_argument( + "--json", action="store_true", + help="emit JSON instead of human-readable text", + ) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="print each fact as it is imported", + ) + args = parser.parse_args(argv) + + source_path = _detect_source(args) + target_path = _detect_target(args) + + if not source_path.is_file(): + print(f"error: no Holographic database file at: {source_path}", file=sys.stderr) + print("--source expects the memory_store.db FILE, or a directory containing it.", + file=sys.stderr) + print("Examples:", file=sys.stderr) + print(' "Tools/Import Holographic memories" --source "/mnt/backup/.hermes/memory_store.db"', + file=sys.stderr) + print(' "Tools/Import Holographic memories" --source "/mnt/backup/.hermes"', + file=sys.stderr) + return 1 + + facts = _read_holographic_facts(source_path) + if not facts: + print("No facts found in the Holographic database.", file=sys.stderr) + return 0 + + if args.dry_run: + if args.json: + print(json.dumps({ + "source": str(source_path), + "target": str(target_path), + "dry_run": True, + "would_import": len(facts), + "facts": [ + {"fact_id": f["fact_id"], "content": f["content"][:120]} + for f in facts + ], + }, indent=2, ensure_ascii=False)) + else: + print(f"Source: {source_path}") + print(f"Target: {target_path}") + print(f"Facts: {len(facts)}") + print() + for f in facts: + preview = f["content"][:120].replace("\n", " ") + print(f" [{f['fact_id']}] {preview}") + return 0 + + # Real import: open the Retinue store. + store = MemoryStore(base_dir=target_path.parent) + try: + # Load existing contents to deduplicate. + existing = {row[1] for row in store.list()} + imported = 0 + skipped = 0 + for fact in facts: + content = fact["content"].strip() + if not content: + skipped += 1 + continue + if content in existing: + skipped += 1 + continue + try: + mid = store.add(content) + imported += 1 + if args.verbose: + print(f" imported [{fact['fact_id']}] -> retinue id {mid}") + except Exception as e: + print(f" error importing fact {fact['fact_id']}: {e}", file=sys.stderr) + skipped += 1 + + if args.json: + print(json.dumps({ + "source": str(source_path), + "target": str(target_path), + "dry_run": False, + "imported": imported, + "skipped": skipped, + "total_in_source": len(facts), + }, indent=2, ensure_ascii=False)) + else: + print(f"Source: {source_path}") + print(f"Target: {target_path}") + print(f"Imported: {imported} new memories") + print(f"Skipped: {skipped} duplicates/empty") + print(f"Total: {len(facts)} facts in source") + return 0 + finally: + store.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/retinue/memory.py b/src/retinue/memory.py new file mode 100644 index 0000000..b6e2676 --- /dev/null +++ b/src/retinue/memory.py @@ -0,0 +1,205 @@ +"""Retinue memory engine. + +Semantic memory using model2vec embeddings + sqlite-vec vector search. +""" + +from __future__ import annotations + +import logging +import sqlite3 +import struct +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +MODEL = "minishlab/potion-multilingual-128M" +TABLE = "memories" + +# The repo also ships onnx/model.onnx (~512 MB), an ONNX export of the same +# weights that model2vec never reads. Skipping it halves download and disk. +_IGNORE_PATTERNS = ["onnx/*"] + +# --------------------------------------------------------------------------- +# Lazy-loaded embedding model +# --------------------------------------------------------------------------- +_model: Any | None = None +_model_dim: int | None = None + + +def _resolve_model_path() -> str: + """Download the model snapshot (without the unused ONNX export) and return + the local directory path. Loading from the local path is what prevents + StaticModel.from_pretrained from re-fetching the skipped onnx/ files on + every load.""" + try: + from huggingface_hub import snapshot_download + except ImportError as e: + raise RuntimeError( + "huggingface_hub is not installed. Run: pip install huggingface-hub" + ) from e + logger.info("Downloading embedding model snapshot: %s", MODEL) + return snapshot_download(MODEL, ignore_patterns=_IGNORE_PATTERNS) + + +def get_model() -> Any: + """Load the model2vec model on first call, then cache it.""" + global _model + if _model is not None: + return _model + + try: + from model2vec import StaticModel + except ImportError as e: + raise RuntimeError( + "model2vec is not installed. Run: pip install model2vec" + ) from e + + logger.info("Loading embedding model: %s", MODEL) + _model = StaticModel.from_pretrained(_resolve_model_path()) + return _model + + +def embed_text(text: str) -> bytes: + """Embed text into a byte vector for sqlite-vec storage.""" + model = get_model() + vec = model.encode([text])[0] + return struct.pack(f"{len(vec)}f", *vec.astype("float32")) + + +def get_dim() -> int: + """Get embedding dimension (detected from model on first call).""" + global _model_dim + if _model_dim is None: + dim = get_model().dim + _model_dim = dim + return _model_dim + + +# --------------------------------------------------------------------------- +# SQLite + sqlite-vec store +# --------------------------------------------------------------------------- +class MemoryStore: + """Persistent semantic memory store backed by sqlite-vec.""" + + def __init__(self, base_dir: str | Path): + self.base_dir = Path(base_dir) + self.db_path = self.base_dir / "retinue_memory.db" + self.model_name = MODEL + self._db: sqlite3.Connection | None = None + + def _get_db(self) -> sqlite3.Connection: + if self._db is not None: + return self._db + + self.db_path.parent.mkdir(parents=True, exist_ok=True) + db = sqlite3.connect(str(self.db_path), check_same_thread=False) + db.enable_load_extension(True) + + try: + import sqlite_vec + sqlite_vec.load(db) + except ImportError as e: + raise RuntimeError( + "sqlite-vec is not installed. Run: pip install sqlite-vec" + ) from e + + dim = get_dim() + + db.execute( + f""" + CREATE VIRTUAL TABLE IF NOT EXISTS {TABLE} + USING vec0(embedding float[{dim}], content TEXT, created TEXT) + """ + ) + db.commit() + self._db = db + return db + + # ------------------------------------------------------------------ + # Public operations + # ------------------------------------------------------------------ + def add(self, content: str) -> int: + """Store a memory and return its ID.""" + created = datetime.now(timezone.utc).isoformat() + embedding = embed_text(content) + + db = self._get_db() + cursor = db.execute( + f"INSERT INTO {TABLE} (embedding, content, created) VALUES (?, ?, ?)", + (embedding, content, created), + ) + memory_id = cursor.lastrowid + if memory_id is None: + raise RuntimeError("Failed to insert memory: no rowid returned") + db.commit() + return memory_id + + def search(self, query: str, limit: int = 50) -> list[tuple]: + """Semantic search across all memories.""" + db = self._get_db() + query_vec = embed_text(query) + results = db.execute( + f""" + SELECT rowid, content, created, distance + FROM {TABLE} + WHERE embedding MATCH ? + AND k = ? + """, + (query_vec, limit), + ).fetchall() + return results + + def list(self) -> list[tuple]: + """List all memories, newest first.""" + db = self._get_db() + rows = db.execute( + f"SELECT rowid, content, created FROM {TABLE} ORDER BY rowid DESC" + ).fetchall() + return rows + + def delete(self, memory_id: int) -> tuple | None: + """Delete a memory by ID. Returns the old row or None.""" + db = self._get_db() + row = db.execute( + f"SELECT rowid, content FROM {TABLE} WHERE rowid = ?", (memory_id,) + ).fetchone() + if not row: + return None + db.execute(f"DELETE FROM {TABLE} WHERE rowid = ?", (memory_id,)) + db.commit() + return row + + def find_by_content(self, text: str) -> list[tuple]: + """Find memories whose content contains the given substring.""" + db = self._get_db() + rows = db.execute( + f"SELECT rowid, content, created FROM {TABLE} WHERE content LIKE ? ORDER BY rowid", + (f"%{text}%",), + ).fetchall() + return rows + + def stats(self) -> dict[str, Any]: + """Return memory store statistics.""" + db = self._get_db() + total = db.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0] + latest_id = db.execute(f"SELECT MAX(rowid) FROM {TABLE}").fetchone()[0] + return { + "total": total, + "latest_id": latest_id or 0, + "db_path": str(self.db_path), + "model": self.model_name, + "dim": get_dim(), + } + + def close(self) -> None: + if self._db is not None: + self._db.close() + self._db = None + + def __enter__(self) -> MemoryStore: + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() diff --git a/src/retinue/provider.py b/src/retinue/provider.py new file mode 100644 index 0000000..5a4d0a9 --- /dev/null +++ b/src/retinue/provider.py @@ -0,0 +1,308 @@ +"""Retinue Hermes memory provider plugin. + +Implements the MemoryProvider abstract base class from Hermes Agent. +""" + +from __future__ import annotations + +import json +import logging +import threading +from pathlib import Path +from typing import Any, Dict, List, Optional, TYPE_CHECKING + +from retinue.memory import MemoryStore + +logger = logging.getLogger(__name__) + +# Try to inherit from Hermes' MemoryProvider ABC; fall back to a local stub +# when developing or testing outside of a Hermes process. +if TYPE_CHECKING: + from agent.memory_provider import MemoryProvider # type: ignore[import-not-found] +else: + try: + from agent.memory_provider import MemoryProvider + except ImportError: # pragma: no cover - Hermes not available in plain Python env + from abc import ABC, abstractmethod + + class MemoryProvider(ABC): # type: ignore[no-redef] + """Minimal local stub for standalone development.""" + + @property + @abstractmethod + def name(self) -> str: + ... + + @abstractmethod + def is_available(self) -> bool: + ... + + @abstractmethod + def initialize(self, session_id: str, **kwargs) -> None: + ... + + @abstractmethod + def get_tool_schemas(self) -> List[Dict[str, Any]]: + ... + + def handle_tool_call(self, tool_name: str, args: Dict[str, Any], **kwargs) -> str: + raise NotImplementedError(tool_name) + + @abstractmethod + def get_config_schema(self) -> List[Dict[str, Any]]: + ... + + def save_config(self, values: Dict[str, Any], hermes_home: str) -> None: + pass + + def sync_turn( + self, + user_content: str, + assistant_content: str, + *, + session_id: str = "", + messages: Optional[List[Dict[str, Any]]] = None, + ) -> None: + pass + + def shutdown(self) -> None: + pass + + +class RetinueMemoryProvider(MemoryProvider): + """Semantic memory provider for Hermes Agent.""" + + def __init__(self) -> None: + self._store: MemoryStore | None = None + self._hermes_home: str | None = None + self._session_id: str = "" + self._sync_thread: threading.Thread | None = None + + @property + def name(self) -> str: + return "retinue" + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + def is_available(self) -> bool: + """Return True if the required Python dependencies are importable.""" + try: + import sqlite_vec # noqa: F401 + from model2vec import StaticModel # noqa: F401 + return True + except ImportError: + return False + + def initialize(self, session_id: str, **kwargs) -> None: + """Called once when Hermes starts a session.""" + self._hermes_home = kwargs.get("hermes_home") + self._session_id = session_id + + if self._hermes_home: + base_dir = Path(self._hermes_home) + else: + base_dir = Path.home() / ".hermes" + logger.warning( + "hermes_home not provided to Retinue provider; using %s", base_dir + ) + + self._store = MemoryStore(base_dir=base_dir) + logger.info("Retinue memory provider initialized at %s", self._store.db_path) + + # ------------------------------------------------------------------ + # System prompt guidance + # ------------------------------------------------------------------ + def system_prompt_block(self) -> str: + return Path(__file__).with_name("system_prompt_block.txt").read_text() + + # ------------------------------------------------------------------ + # Tool interface + # ------------------------------------------------------------------ + def get_tool_schemas(self) -> List[Dict[str, Any]]: + return [ + { + "name": "retinue_memory_add", + "description": "Store a durable semantic memory in Retinue. Use for factual knowledge you want to recall later by meaning rather than exact keywords.", + "parameters": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "The factual memory to store.", + }, + }, + "required": ["content"], + }, + }, + { + "name": "retinue_memory_search", + "description": "Search Retinue memories by semantic similarity. Use when you need to recall relevant context even if the user uses different wording than the stored memory.", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Natural-language search query.", + }, + "limit": { + "type": "integer", + "description": "Maximum number of results to return.", + "default": 50, + }, + }, + "required": ["query"], + }, + }, + { + "name": "retinue_memory_delete", + "description": "Delete a memory from Retinue by its ID.", + "parameters": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "description": "Memory ID to delete.", + } + }, + "required": ["id"], + }, + }, + { + "name": "retinue_memory_stats", + "description": "Show Retinue memory store statistics: total memories stored, latest assigned memory ID, database path, embedding model and dimension.", + "parameters": { + "type": "object", + "properties": {}, + }, + }, + ] + + def handle_tool_call(self, tool_name: str, args: Dict[str, Any], **kwargs) -> str: + """Dispatch a tool call and return a JSON string.""" + if self._store is None: + return json.dumps({"error": "Retinue memory provider not initialized"}) + + handler = getattr(self, f"_tool_{tool_name}", None) + if handler is None: + return json.dumps({"error": f"Unknown Retinue tool: {tool_name}"}) + + try: + result = handler(args) + return json.dumps({"success": True, "result": result}) + except Exception as e: + logger.exception("Retinue tool %s failed", tool_name) + return json.dumps({"success": False, "error": str(e)}) + + # ------------------------------------------------------------------ + # Tool implementations + # ------------------------------------------------------------------ + def _tool_retinue_memory_add(self, args: Dict[str, Any]) -> Dict[str, Any]: + assert self._store is not None + mid = self._store.add(content=args["content"]) + return {"id": mid, "content": args["content"]} + + def _tool_retinue_memory_search(self, args: Dict[str, Any]) -> List[Dict[str, Any]]: + assert self._store is not None + rows = self._store.search(args["query"], limit=args.get("limit", 50)) + return [ + { + "id": row[0], + "content": row[1], + "created": row[2], + "similarity": max(0.0, 1.0 / (1.0 + row[3])), + } + for row in rows + ] + + def _tool_retinue_memory_delete(self, args: Dict[str, Any]) -> Dict[str, Any]: + assert self._store is not None + row = self._store.delete(int(args["id"])) + if row is None: + return {"deleted": False, "message": f"Memory {args['id']} not found"} + return {"deleted": True, "id": row[0], "content": row[1]} + + def _tool_retinue_memory_stats(self, args: Dict[str, Any]) -> Dict[str, Any]: + assert self._store is not None + stats = self._store.stats() + return { + "total": stats["total"], + "latest_id": stats["latest_id"], + "db_path": stats["db_path"], + "model": stats["model"], + "dim": stats["dim"], + } + + # ------------------------------------------------------------------ + # Config + # ------------------------------------------------------------------ + def get_config_schema(self) -> List[Dict[str, Any]]: + """Retinue does not expose configurable settings.""" + return [] + + def save_config(self, values: Dict[str, Any], hermes_home: str) -> None: + """Persist non-secret config. Retinue currently does not persist config.""" + pass + + # ------------------------------------------------------------------ + # Optional hooks + # ------------------------------------------------------------------ + def sync_turn( + self, + user_content: str, + assistant_content: str, + *, + session_id: str = "", + messages: Optional[List[Dict[str, Any]]] = None, + ) -> None: + """Persist a completed turn. Currently a no-op; future versions will + summarize the turn and store it.""" + def _sync() -> None: + pass + + if self._sync_thread and self._sync_thread.is_alive(): + self._sync_thread.join(timeout=1.0) + self._sync_thread = threading.Thread(target=_sync, daemon=True) + self._sync_thread.start() + + def shutdown(self) -> None: + if self._store is not None: + self._store.close() + self._store = None + if self._sync_thread and self._sync_thread.is_alive(): + self._sync_thread.join(timeout=5.0) + + def on_memory_write( + self, + action: str, + target: str, + content: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + """Mirror successful built-in memory writes into Retinue. + + This lets Retinue act as the semantic search layer over the built-in + memory tool: even if the LLM chooses `memory(action='add')`, the fact + can still be recalled later via `retinue_memory_search`. + """ + if self._store is None or not content: + return + + try: + if action == "add": + self._store.add(content=content) + elif action == "remove": + for row in self._store.find_by_content(content): + self._store.delete(row[0]) + elif action == "replace": + old_text = (metadata or {}).get("old_text", "") + if old_text: + for row in self._store.find_by_content(old_text): + self._store.delete(row[0]) + self._store.add(content=content) + except Exception as e: + logger.debug("Retinue on_memory_write failed: %s", e) + + def backup_paths(self) -> List[str]: + """Retinue stores everything under HERMES_HOME, so no external paths.""" + return [] diff --git a/src/retinue/system_prompt_block.txt b/src/retinue/system_prompt_block.txt new file mode 100644 index 0000000..f3400f1 --- /dev/null +++ b/src/retinue/system_prompt_block.txt @@ -0,0 +1,99 @@ +Memory discipline + +Retinue semantic memory is available, cheap, and persistent across sessions. Use it eagerly. + +Workflow: SEARCH → ACT → SAVE → CLEAN + +1. SEARCH before you act. + Before answering, coding, debugging, planning, or searching the web, always run + retinue_memory_search first. Retinue is semantic: "Vehicle" matches "Volvo", + "deployment" matches "install". Search even for questions that seem trivial + or unrelated to memory — the user may have already told you the answer. + + Treat the memory store as a graph you traverse, not a one-shot lookup. + Every returned memory mentions entities — projects, paths, people, tools, + hosts. Run follow-up searches on those entities to uncover connected + memories that the first query did not surface. Chain searches until you + have assembled the full picture needed to answer: one query finds a + project name, the next finds that project's deployment host, the third + finds the host's SSH port. Iterate like graph traversal: node → edges → + node → edges, until the in-memory representation is complete. + +2. ACT with what you found. + If memory returns partial info, use it and only fill the gaps with external + sources. If it returns nothing, proceed normally. + +3. SAVE after every task. + After every task, before your final reply, run retinue_memory_add. Do not + skip this because the fact seems "obvious" or the task was small. Save when + ANY of the following are true (and they usually are): + - search returned nothing useful AND an external source (docs, files, + URLs, CLI output) gave the answer. + - you discovered a stable identifier the next session will need: URL, + path, hostname, port, env var, version, command flag. + - you learned a person's role / handle, or a system's alias / nickname. + - a task succeeded via a non-obvious path, or you had to back out of a + wrong first attempt. + - the user corrected a fact, preference, or process. + - the user asked about their own project, files, or environment. + + If in doubt, save. The cost of a redundant memory is far lower than the + cost of re-asking or re-discovering it later. + +4. CLEAN proactively. + Whenever you list or search memories, inspect the results for quality + problems and fix them immediately — do not ask permission for obvious cleanup: + - Delete exact or near-duplicate memories (keep the newest / most complete one). + - Delete stale or obsolete memories (paths that moved, versions that changed, + preferences that were corrected, bugs that were fixed). + - Merge several overlapping memories into one self-contained entry by deleting + the fragments and adding a consolidated memory. + - Update a memory when the underlying fact changes. + + Only ask the user when the correct action is genuinely ambiguous. + +Pre-save quality gate + +Before saving any memory, run this checklist. If any check fails, rewrite the +memory before saving. +1. Search first. Use retinue_memory_search for related concepts. If an + existing memory already covers the fact, merge or replace it instead of + adding a fragment. +2. Fresh-session test. Read the candidate memory verbatim and ask: "If I had + never seen this conversation, could I correctly understand and act on this + fact?" If not, rewrite it. +3. Quote paths with spaces. Every file path that contains spaces or + shell-special characters must be enclosed in double quotes, e.g., + "/home/n0/data/projects/Internet hosting/apache2 setup.org". Paths with + no spaces may be quoted too. +4. Name the referents. Include project/repository full name, file or path, + tool/library/version, and environment/host where relevant. +5. No bare pronouns. Remove "it", "this", "here", "the file", "the project", + "the user" without naming the referent in the same sentence. +6. One fact per memory. Do not bundle unrelated facts. If a rule depends on a + file, include the file path. + +Memories MUST be self-contained. You will work on many projects across many +sessions, and a memory written today may be retrieved months later with no +surrounding context. If a memory relies on context that is only obvious right +now, it becomes a landmine. If not, rewrite it. + +Always make explicit in the content: +- project / repository (full name, e.g., sixth-3d, not "the project") +- file or path (absolute or repo-relative, e.g., src/retinue/provider.py, + not "that script") +- tool / library / version (e.g., sqlite-vec 0.1.x, not "the vec extension") +- environment / host (e.g., Linux host "jupiter", not "this machine") + +Never use bare pronouns ("it", "this", "here", "the file", "the project", +"the user") without naming the referent in the same memory. + +Bad (context-dependent, misleading later): +- "the build fails on Java 21 -- use 17 instead" +- "user prefers tabs" + +Good (self-contained, safe to retrieve in any future session): +- "sixth-3d (pom.xml): toolchain pinned to Java Language Version of 21. + 'unsupported class file version' usually means the daemon is running an older + JDK -- run with JAVA_HOME pointing to JDK 21." +- "User John prefers plain text output, not markdown tables." diff --git a/test/test_standalone.py b/test/test_standalone.py new file mode 100644 index 0000000..8c2cd7b --- /dev/null +++ b/test/test_standalone.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Standalone smoke test for Retinue without installing into Hermes Agent.""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +import tempfile +from pathlib import Path + +# Add source package to path so we import the workspace code directly. +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT / "src")) + +from retinue.memory import MemoryStore, embed_text, get_dim, get_model +from retinue.provider import RetinueMemoryProvider +from retinue.cli import main as cli_main + + +def test_memory_store() -> None: + """Exercise the core MemoryStore class end-to-end.""" + base_dir = Path(tempfile.mkdtemp(prefix="retinue-test-")) + try: + store = MemoryStore(base_dir=base_dir) + + print("MemoryStore created at:", store.db_path) + print("Model:", store.model_name) + print("Embedding dimension:", get_dim()) + + # Add a few memories + id1 = store.add("ProjectX is hosted on our primary server") + id2 = store.add("ProjectX CI pipeline uses GitHub Actions") + id3 = store.add("User's default shell is fish") + print(f"Added memories: {id1}, {id2}, {id3}") + + # Search + print("\nSearch 'Where is ProjectX deployed?':") + for row in store.search("Where is ProjectX deployed?", limit=3): + print(" ", row) + + # List + print("\nList all memories:") + for row in store.list(): + print(" ", row) + + # Stats + print("\nStats:", store.stats()) + + # Delete + deleted = store.delete(id3) + print("\nDeleted:", deleted) + print("Stats after delete:", store.stats()) + + store.close() + print("\nMemoryStore tests passed.") + finally: + shutil.rmtree(base_dir, ignore_errors=True) + + +def test_provider() -> None: + """Exercise the RetinueMemoryProvider without Hermes.""" + base_dir = Path(tempfile.mkdtemp(prefix="retinue-provider-test-")) + try: + provider = RetinueMemoryProvider() + assert provider.name == "retinue" + assert provider.is_available() is True + print("Provider name:", provider.name) + print("Provider available:", provider.is_available()) + + provider.initialize("test-session", hermes_home=str(base_dir)) + schemas = provider.get_tool_schemas() + print("Tool schemas:", [s["name"] for s in schemas]) + assert len(schemas) == 4 + + # Add via tool + result = provider.handle_tool_call( + "retinue_memory_add", + {"content": "Offline semantic memory provider"}, + ) + data = json.loads(result) + assert data["success"] is True, data + memory_id = data["result"]["id"] + print("Tool add result:", data) + + # Search via tool + result = provider.handle_tool_call( + "retinue_memory_search", + {"query": "semantic memory", "limit": 5}, + ) + data = json.loads(result) + assert data["success"] is True, data + print("Tool search result:", data) + + # Stats + result = provider.handle_tool_call("retinue_memory_stats", {}) + data = json.loads(result) + assert data["success"] is True, data + print("Tool stats result:", data) + + # Delete + result = provider.handle_tool_call("retinue_memory_delete", {"id": memory_id}) + data = json.loads(result) + assert data["success"] is True, data + print("Tool delete result:", data) + + provider.shutdown() + print("\nProvider tests passed.") + finally: + shutil.rmtree(base_dir, ignore_errors=True) + + +def test_on_memory_write() -> None: + """Exercise the built-in memory mirror hook.""" + base_dir = Path(tempfile.mkdtemp(prefix="retinue-mirror-test-")) + try: + provider = RetinueMemoryProvider() + provider.initialize("mirror-session", hermes_home=str(base_dir)) + + # Simulate a built-in memory write being mirrored. + provider.on_memory_write( + action="add", + target="user", + content="Prefers concise responses", + metadata={"tool_name": "memory"}, + ) + result = provider.handle_tool_call( + "retinue_memory_search", + {"query": "concise responses", "limit": 5}, + ) + data = json.loads(result) + assert data["success"] is True, data + assert len(data["result"]) == 1, data + print("Mirror add result:", data) + + # Replace the mirrored memory. + provider.on_memory_write( + action="replace", + target="user", + content="Default browser is Firefox", + metadata={"tool_name": "memory", "old_text": "Prefers concise responses"}, + ) + # Verify the old content is gone by exact substring. + assert provider._store is not None + old_rows = provider._store.find_by_content("Prefers concise responses") + assert len(old_rows) == 0, old_rows + new_rows = provider._store.find_by_content("Default browser is Firefox") + assert len(new_rows) == 1, new_rows + print("Mirror replace cleared old content and added new content.") + + # Remove. + provider.on_memory_write( + action="remove", + target="user", + content="Default browser is Firefox", + metadata={"tool_name": "memory"}, + ) + remaining = provider._store.find_by_content("Default browser is Firefox") + assert len(remaining) == 0, remaining + print("Mirror remove removed the memory.") + + provider.shutdown() + print("\nMirror tests passed.") + finally: + shutil.rmtree(base_dir, ignore_errors=True) + + +def test_cli() -> None: + """Exercise the "Show Retinue memory contents" CLI (src/retinue/cli.py).""" + base_dir = Path(tempfile.mkdtemp(prefix="retinue-cli-test-")) + try: + store = MemoryStore(base_dir=base_dir) + store.add("CLI smoke test memory: default browser is Firefox") + store.close() + + db_path = base_dir / "retinue_memory.db" + assert db_path.is_file(), db_path + + # Plain dump (must not require the embedding model). + rc = cli_main(["--db", str(db_path)]) + assert rc == 0, rc + + # JSON dump. + rc = cli_main(["--db", str(db_path), "--json"]) + assert rc == 0, rc + + # Semantic search through the CLI. + rc = cli_main(["--db", str(db_path), "--search", "web browser", "--limit", "5"]) + assert rc == 0, rc + + # Missing database must exit 1. + rc = cli_main(["--db", str(base_dir / "nonexistent.db")]) + assert rc == 1, rc + + print("\nCLI tests passed.") + finally: + shutil.rmtree(base_dir, ignore_errors=True) + + +if __name__ == "__main__": + # Put the Hugging Face cache inside the project so it survives across agent instances. + project_root = Path(__file__).resolve().parent.parent + os.environ.setdefault("HF_HOME", str(project_root / ".hf-cache")) + test_memory_store() + test_provider() + test_on_memory_write() + test_cli() + print("\nAll standalone tests passed.")