Nemotron advisor and RAG

sudo apt install -y cmake build-essential
cd ~ && git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp
cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=87 -DCMAKE_CUDA_COMPILER=/usr/local/cuda-12.6/bin/nvcc
cmake --build build --config Release -j6
ls build/bin/llama-server

tmux new -s llm
~/llama.cpp/build/bin/llama-server -m ~/sutraai/models/NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf \
  --alias nemotron --n-gpu-layers 999 --ctx-size 8192 --host 127.0.0.1 --port 8080
# leave it running; detach with Ctrl+B then D

curl -s http://127.0.0.1:8080/v1/chat/completions -H "Content-Type: application/json" \
  -d '{"model":"nemotron","messages":[{"role":"user","content":"Say hello in five words."}]}'

import json, time
from pathlib import Path
from openai import OpenAI
ROOT = Path.home() / "sutraai"
client = OpenAI(base_url="http://127.0.0.1:8080/v1", api_key="not-needed")

def advise(metrics_path):
    metrics = json.load(open(metrics_path))
    prompt = ("You are a training advisor for small edge-AI models. Using ONLY the numbers in this JSON, "
              "explain the results to a beginner in 5 short bullet points, then suggest the 2 most useful next steps. "
              "Never invent numbers.\n\n" + json.dumps(metrics))
    t = time.perf_counter()
    r = client.chat.completions.create(model="nemotron", temperature=0.2,
                                       messages=[{"role": "user", "content": prompt}])
    dt = time.perf_counter() - t
    text = r.choices[0].message.content
    toks = r.usage.completion_tokens if r.usage else None
    return text, dt, toks

text, dt, toks = advise(ROOT / "reports/audio_v1/metrics.json")
print(text); print(f"\n{toks} tokens in {dt:.1f} s" + (f" = {toks/dt:.1f} tokens/s" if toks else ""))

import numpy as np
from pathlib import Path
from openai import OpenAI
from sentence_transformers import SentenceTransformer
ROOT = Path.home() / "sutraai"

def chunk_markdown(text, max_chars=1500):
    parts, cur = [], ""
    for para in text.split("\n\n"):
        if len(cur) + len(para) > max_chars and cur:
            parts.append(cur.strip()); cur = ""
        cur += para + "\n\n"
    if cur.strip(): parts.append(cur.strip())
    return parts

sources = [ROOT / "BUILDLOG.md"] + sorted((ROOT / "docs").glob("*.md"))     # copy the SutraAI .md docs into ~/sutraai/docs
chunks, origin = [], []
for src in sources:
    if src.exists():
        for c in chunk_markdown(src.read_text()):
            chunks.append(c); origin.append(src.name)

emb = SentenceTransformer("BAAI/bge-small-en-v1.5", device="cuda")    # small and fast; try NVIDIA's embedder next
E = emb.encode(chunks, normalize_embeddings=True)

def retrieve(q, k=4):
    s = E @ emb.encode([q], normalize_embeddings=True)[0]
    return list(s.argsort()[::-1][:k])

client = OpenAI(base_url="http://127.0.0.1:8080/v1", api_key="not-needed")
def ask(q):
    ids = retrieve(q)
    context = "\n\n".join(f"[{n+1}] ({origin[i]}) {chunks[i]}" for n, i in enumerate(ids))
    prompt = (f"Answer the question using ONLY the numbered sources. Cite them like [1]. "
              f"If the sources don't contain the answer, say so.\n\nSources:\n{context}\n\nQuestion: {q}")
    r = client.chat.completions.create(model="nemotron", temperature=0.1, messages=[{"role": "user", "content": prompt}])
    return r.choices[0].message.content, [origin[i] for i in ids]

print(ask("How do I convert a model to int8 TFLite?"))

# 10 questions you know the answer to + a phrase that must appear in the right chunk
tests = [
    ("Which power mode do I use for training?", "MAXN"),
    ("How big should swap be?", "16"),
    ("What is the int8 accuracy budget?", "3 points"),
    # ... add 7 more from your own BUILDLOG
]
hits = sum(any(phrase.lower() in chunks[i].lower() for i in retrieve(q)) for q, phrase in tests)
print(f"hit@4 = {hits}/{len(tests)}")