1 files changed, 41 insertions, 0 deletions
diff --git a/utils.py b/utils.py
index aea653b..6fd3487 100644
--- a/utils.py
+++ b/utils.py
@@ -36,3 +36,44 @@ def tail_lines(path: Path, n: int) -> list[str]:
                 break
             block_size *= 2
     return [l.decode("utf-8", errors="ignore").strip() for l in lines[-n:] if l.strip()]
+
+
+CHUNK_LINES = 4  # consecutive messages per rag chunk
+MIN_CHUNK_CHARS = 40  # windows shorter than this are basically "lol" "yeah", not worth embedding
+SKIP_FRAGMENTS = ("sent sticker:", "sent media:", "sent a random dm to ", "saved media:")
+
+
+def make_chunks(lines: list[str]) -> list[str]:
+    # rag chunks are windows of consecutive messages instead of single lines, one-liners embed badly.
+    # the step is 1 so when the tail of memory.txt slides, the old windows keep the exact same text and don't get re-embedded
+    kept = []
+    for l in lines:
+        c = l.strip()
+        if any(f in c for f in SKIP_FRAGMENTS):
+            continue
+        if "said:" in c and c.split("said:", 1)[1].strip() == "":
+            continue
+        kept.append(c)
+    if len(kept) <= CHUNK_LINES:
+        windows = ["\n".join(kept)] if kept else []
+    else:
+        windows = ["\n".join(kept[i:i + CHUNK_LINES]) for i in range(len(kept) - CHUNK_LINES + 1)]
+    return [w for w in windows if len(w) >= MIN_CHUNK_CHARS]
+
+
+def pick_chunks(scored: list, top_n: int, threshold: float) -> list:
+    # scored is (chunk, similarity) sorted best first. drops anything under the threshold and skips windows
+    # that share a line with one already picked (they overlap, otherwise the top 5 would be the same messages)
+    picked = []
+    used = set()
+    for chunk, sim in scored:
+        if sim < threshold:
+            break
+        lines = set(chunk.split("\n"))
+        if lines & used:
+            continue
+        used |= lines
+        picked.append((chunk, sim))
+        if len(picked) >= top_n:
+            break
+    return picked