From 9f82a0a2231ce732cb647ae67d3ac0cd6312c1a4 Mon Sep 17 00:00:00 2001 From: david Date: Wed, 14 Oct 2026 20:45:52 -0300 Subject: hello --- utils.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) (limited to 'utils.py') diff --git a/utils.py b/utils.py index aea653b..6fd3487 100644 --- a/utils.py +++ b/utils.py @@ -36,3 +36,44 @@ def tail_lines(path: Path, n: int) -> list[str]: break block_size *= 2 return [l.decode("utf-8", errors="ignore").strip() for l in lines[-n:] if l.strip()] + + +CHUNK_LINES = 4 # consecutive messages per rag chunk +MIN_CHUNK_CHARS = 40 # windows shorter than this are basically "lol" "yeah", not worth embedding +SKIP_FRAGMENTS = ("sent sticker:", "sent media:", "sent a random dm to ", "saved media:") + + +def make_chunks(lines: list[str]) -> list[str]: + # rag chunks are windows of consecutive messages instead of single lines, one-liners embed badly. + # the step is 1 so when the tail of memory.txt slides, the old windows keep the exact same text and don't get re-embedded + kept = [] + for l in lines: + c = l.strip() + if any(f in c for f in SKIP_FRAGMENTS): + continue + if "said:" in c and c.split("said:", 1)[1].strip() == "": + continue + kept.append(c) + if len(kept) <= CHUNK_LINES: + windows = ["\n".join(kept)] if kept else [] + else: + windows = ["\n".join(kept[i:i + CHUNK_LINES]) for i in range(len(kept) - CHUNK_LINES + 1)] + return [w for w in windows if len(w) >= MIN_CHUNK_CHARS] + + +def pick_chunks(scored: list, top_n: int, threshold: float) -> list: + # scored is (chunk, similarity) sorted best first. drops anything under the threshold and skips windows + # that share a line with one already picked (they overlap, otherwise the top 5 would be the same messages) + picked = [] + used = set() + for chunk, sim in scored: + if sim < threshold: + break + lines = set(chunk.split("\n")) + if lines & used: + continue + used |= lines + picked.append((chunk, sim)) + if len(picked) >= top_n: + break + return picked -- cgit v1.3-2-g0d8e