Add story-to-pack research and a structured situation review
Research toward building a content pack from a story corpus, kept on its own branch and independent of the game. Records the selection experiments against blind labels, and settles selection as gate G2 followed by a human review: review.py writes REVIEW.md and a review.json form, apply_review.py checks the filled form and writes situations.json for the next stage. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01C6UDQ9o6L6Ey173U7XVou6
This commit is contained in:
co-authored by
Claude Opus 5
parent
d84ae495f4
commit
fa3769d0fe
@@ -0,0 +1,34 @@
|
||||
"""Embed chunks on the local endpoint, checkpointing as we go."""
|
||||
import json, pathlib, ssl, time, urllib.request
|
||||
|
||||
URL = 'https://inference.lan:8443/v1/embeddings' # placeholder: the real host is never committed
|
||||
MODEL = 'nomic-embed-text'
|
||||
BATCH = 24
|
||||
ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
chunks = json.loads(pathlib.Path('chunks.json').read_text(encoding='utf-8'))
|
||||
out = pathlib.Path('embeddings.json')
|
||||
done = json.loads(out.read_text()) if out.exists() else []
|
||||
|
||||
start = len(done)
|
||||
print(f'{len(chunks)} chunks, resuming at {start}', flush=True)
|
||||
t0 = time.time()
|
||||
for i in range(start, len(chunks), BATCH):
|
||||
batch = [c['text'] for c in chunks[i:i + BATCH]]
|
||||
body = json.dumps({'model': MODEL, 'input': batch}).encode()
|
||||
req = urllib.request.Request(URL, data=body, headers={'Content-Type': 'application/json'})
|
||||
for attempt in range(4):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=180, context=ctx) as r:
|
||||
d = json.load(r)
|
||||
done.extend(e['embedding'] for e in d['data'])
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt == 3: raise
|
||||
print(f' retry {attempt+1} at {i}: {e}', flush=True)
|
||||
time.sleep(3 * (attempt + 1))
|
||||
out.write_text(json.dumps(done))
|
||||
el = time.time() - t0
|
||||
n = len(done)
|
||||
print(f' {n}/{len(chunks)} {el:.0f}s ({n/max(el,1):.1f}/s)', flush=True)
|
||||
print(f'done: {len(done)} embeddings in {time.time()-t0:.0f}s')
|
||||
Reference in New Issue
Block a user