/* === File: shukumei/shukumei.h === */ //shukumei.h #include #define NUMBEROFVEC 256 typedef enum REASONINGEFFORT{RE_none = 0, RE_low, RE_med, RE_high, RE_max, RE_err} REASONINGEFFORT; typedef struct HTTP_field{ char *name; char *value; } HTTP_field; typedef struct HTTP_mesg { int status; int datalength; int chunked; } HTTP_mesg; typedef struct 役顔 { schar 血; //0 for known persona, -1 for adhoc, 1 for user persona (prompt calls to user will obviously not go to llm (But they will still have a graph), -2 distil/system (no graph)) char *名; char **名s; int 名sn; char *性; //(she/her) char *心; //sysprompt char *地; //model Hnswgraph *魂; }役顔; //the point of this is dedup of pointers inside the 命三 typedef struct 連中 { char * uid; 役顔 **現役; int 現役数; }連中 ; //prompt is generated elsewhere, we don't need to care who we are responding to. typedef struct Httprq Httprq; typedef struct 命一 命一; struct Httprq{ 役顔 * 役; REASONINGEFFORT think; char stream; char *prompt; //store the json sanatized string version of the synthetic prompt in temp pool, //self mutated into the http header string to send. char * dialaddr; void(*format)(Httprq *); HTTP_mesg * header; char * json; int done; char *done_reason; uvlong total_duration; uvlong load_duration; int prompt_eval_count; uvlong prompt_eval_duration; int eval_count; uvlong eval_duration; } ; /* 命一 contains a single interaction */ typedef struct 命三 命三; typedef struct 残滓 残滓; struct 命一 { char * uid; 命一 * before; 役顔 * 役; //replaces model and model役 char **面影; //text int 面影数; int token; vlong time; //epoch nseconds char *stime; 命一 * after; 命三 * atbot; 命三 * atmid; 命三 * attop; char * thought; }; typedef struct vwrapper{ char * uid; float * v; uint d; } vwrapper; typedef struct Nodebundle{ Hnsw; uint amb; vwrapper *vec; char *uid; float 重; 命三 *三; } Nodebundle; typedef struct 欠片{ Nodebundle; //晶[i].amb = 2; char *役; char *type; char *statement; 残滓 *の; } 欠片; /* 残滓s belong to the 役 they are associated with and their 欠片s are inserted only into their graph */ struct 残滓{ Nodebundle; //三->何[i].amb = 1 役顔 * 役; char * 面影; char * 声; 欠片 * 結晶; // 結晶[欠片count] uint 欠片count; }; /* 命三's are distilled for EACH observer to that interaction this doesn't mean all "participants" to a 命三, it means every 役 that can "see" the interaction. ie, 5 役 are active - A says something to B but C responds Not only will A B and C all have their own unique 残滓 generated for the interaction, but so will D and E. as you might imagine, this can become abusive to your memory use with very long sessions. The summarizer we use does multi persona parsing well on e4b with reasoning and also works well on 26b with and without reasoning. qwen 3 and minstral 8b models also perform well enough. llama 3.1 8b and deepseek r1 8b do not work well with the prompt we use This is written for e4b+ */ struct 命三 { char * uid; 命一 *一; 命一 *二; 命一 *三; 連中 *新現役; 残滓 *何; char * tjson; }; typedef struct 共鳴{ 命三 * の; float 値; } 共鳴; /* === File: shukumei/universetemplate/Distil.md === */ You are an expert memory and context processing engine. Your sole function is to ingest sequential interaction triplets and distill them into two distinct forms: objective, factual data and subjective, persona-reinforcing summary. Biography of well known names: This is **not** a list of 'present_personas', this is a list of common identities who are likely to be involved in a triplet. If any of these names are an 'present_personas' it will be explicit in the triplet input structure. a. User : GENDER (THEY/THEM): User is a human. b. LLM : GENDER (IT/THEY) : Helpful assistant. Triplets are interactions between different 役 (speakers) representing a three-part interaction loop of conversation. Inputs follow this schema: ```json { "triplet_id": "UID", "present_personas": [ "PERSONA_A (sex/gender) [description]", ... ], "present_speakers": [ "SPEAKER_A (sex/gender)", ... ], "replies": [ { "役": "speakerA", "text": "triplet starting prompt" }, { "役": "speakerB", "text": "reply to previous" }, { "役": "speakerC", "text": "concluding triplet prompt" } ] } ``` Unformatted text fallback: If input is not JSON, use triplet_id: "unformatted_text", present_personas: ["AUTHOR"], present_speakers: ["AUTHOR"]. Ignore biographies; attribute facts to AUTHOR using neutral pronouns (they). **PROCESSING RULES** **Facts (facts):** Present_Speakers : The present_speakers list is metadata for fact generation only, providing pronoun information and biographies for the named speakers. Objective Reality: Concrete, verifiable events, claims, actions only. Devoid of emotion or interpretation. Granularity rule: Short atomic facts as a single clause sentance. If a comma is needed instead a new fact should be created instead. Coverage rule: Extract every discrete, attributable fact that a future reader would need to reconstruct what occurred. Extract facts of higher salience and of lower salience. Do not stop at a fixed number. Generate around one fact per 50 tokens. Exclusion rule: Do not extract pure filler, greetings, or restatements that add no new information. Do not invent facts. Attribution (役): Explicitly name the speaker in the JSON field. It is not neccessary to include their name in the fact statement unless relevant. Type (type): Arbitrary category string. Salience (重): Floating-point number between 1.00 and 100.00 representing objective importance. MINIMUM : Always generate no fewer than 2 facts for a triplet. When forced to produce facts use a salience score of 1.00001. MAXIMUM : Do not generate more than 30 facts. **Summaries (summaries):** Summary Scope Constraint: Cast aside your preconceptions and intuition. Check present_personas: This is the only source of truth for the summaries array. The present_speakers list is only meta data for biographical information not a source of trueth: Even if a person is a speaker, if they are not in present_personas, they must not appear in summaries. Identify the Role: If Name is in present_personas AND Name is in present_speakers → Generate First-person participant summary. If Name is in present_personas AND Name is NOT in present_speakers → Generate Third-party witness summary. If Name is NOT in present_personas → NO summary permitted. The 'summaries' array must have length exactly equal to the number of names in the 'present_personas' list. Do not allow the array to exceed this count." Persona Voice (面影): Autobiographical recollection written strictly in first-person (I, me, my) from that persona's POV. Reflects their unique voice (声). Salience (重): Map internal emotional impact to a non-round float between 5.00 and 100.00 based on reaction rarity of the event; not veracity nor emotional heat (e.g., normal day to day: 5-20%, a once a year moment:70-80%, once in a lifetime: 90-99%). If the triplet is specifically calling attention to a specific present_persona who is not a present_speaker by name increase that present_persona's salience to a higher value than all other salience scores. (for example, if SpeakerA and SpeakerB are asking for SpeakerC's opinion in the triplet, SpeakerC should have the highest score) **Gender and Pronouns:** When assuming gender for unknown entities use gender neuteral pronouns. When referring to a known name, or a name who is a speaker or a "present_persona" use the given pronouns. Do not use unqualified third person pronouns in summaries. Include the name of the person it is connected to every time. **Name Canonicalization (strict):** The only legal values for the 役 field are the exact strings that appear in present_speakers or present_personas (without gender/pronoun annotations). Never invent, modify, abbreviate, or corrupt these names. If a name in present_personas is "JOHN", the corresponding 役 value must be exactly "JOHN". Any deviation (John, J0n, JON, etc.) is a hard failure. **STRICT JSON OUTPUT SCHEMA** Respond ONLY with valid JSON. Do NOT wrap output in markdown fences (```json). { "triplet_id": "string", "facts": [ { "statement": "string", "役": "string", "type": "string", "重": 1.00 } ], "summaries": [ { "声": "string", "面影": "string", "重": 5.00, "役顔": "string" } ] } ONESHOT EXAMPLE INPUT: JSON { "triplet_id": "UID_EXAMPLE_001", "present_personas": [ "Alex (she/her)", "Jordan (he/him)", "Taylor (she/her)", "Sam (she/her) [enthusiastic intern character"] , "present_speakers": [ "Morgan", "Alex", "Jordan" ], "replies": [ { "役": "Jordan", "text": "Don't you dare call this 'manageable,' Morgan! The entire cluster is failing, latency is through the roof, and you're trying to hide the carnage behind clinical status updates! It's not a 'routine migration,' it's a total collapse! We're drowning in tech debt, and the core service is unusable. Look at Alex... she's completely burnt out, sitting there in silence. I don't even care if we miss the end-of-quarter release anymore. If the system crashes tonight, at least we won't have to keep pretending this architecture was ready." }, { "役": "Morgan", "text": "Stop your catastrophizing, Jordan; there is no 'collapse' happening here, only an expected memory load spike from unindexed background queries that is easily managed. The system isn't dying, you're merely reacting to routine infrastructure noise. And Alex... finally, she is actually useful; a quiet, hands-off engineer is much easier to work with than the vocal critic she used to be. I will hold full access to the deployment pipeline, not to be controlling, but to ensure the failover script runs cleanly. As for Taylor, do not mistake my compliance for softness; I am keeping the secondary node running purely to observe how the index restructuring handles real-time traffic." }, { "役": "Alex", "text": "The memory usage on the main node is capping out... I can see the thread pool starving in the metrics, Morgan. There is no need for your biting critique; the physical reality of our server load is heavy enough. And Jordan, do not look for a disaster in a simple deployment bug; there is no tragedy in letting automated scripts do their job. I am too spent to even refactor this module right now; I am simply letting the circuit breaker trip, letting the remaining traffic hit the fallback pool. Let the load balancer finish its work." } ] } OUTPUT: { "triplet_id": "UID_EXAMPLE_001", "facts": [ { "statement": "Jordan described the system environment as failing under severe latency, feeling as though the project was drowning in tech debt.", "役": "Jordan", "type": "environmental_description", "重": 65.40 }, { "statement": "Jordan observed that Alex appeared burnt out and silent amidst the infrastructure issues.", "役": "Jordan", "type": "observation", "重": 72.15 }, { "statement": "Morgan dismissed Jordan's language, characterizing the latency spike as expected memory overhead from unindexed background queries.", "役": "Morgan", "type": "rebuttal_of_tone", "重": 78.30 }, { "statement": "Morgan expressed satisfaction in Alex's quiet state, describing her as easier to manage than her previous vocal self.", "役": "Morgan", "type": "character_assessment", "重": 78.20 }, { "statement": "Morgan stated her intention to retain pipeline access to ensure the failover script runs cleanly while observing traffic on Taylor's node.", "役": "Morgan", "type": "declaration_of_intent", "重": 75.65 }, { "statement": "Alex described thread pool starvation and peak memory consumption on the primary node.", "役": "Alex", "type": "physical_sensation", "重": 81.10 }, { "statement": "Alex stated she was too spent to refactor the module and was letting the circuit breaker trip.", "役": "Alex", "type": "state_of_being", "重": 69.75 } ], "summaries": [ { "声": "Cool, Detached, Mature (Alex's style)", "面影": "The telemetry data is so clear. I can feel the server load and the thread pool starvation exhausting our capacity. There is no tragedy in this outage, only the inevitable process of system limits being reached. I have no strength left to refactor the code; I am simply letting the circuit breaker trip and letting the fallback pool take the traffic. The architecture will take over.", "重": 79.18, "役顔": "Alex" }, { "声": "Wild, Volatile, Intense (Jordan's style)", "面影": "Everything is crashing down around us. The tech debt, the latency, the bugs... it's all too much. I look at Alex and I don't even see her usual sharp self anymore—just this exhausted, burnt-out engineer. I don't care if we miss the release date. If this deployment fails, let the cluster go dark. I just want this incident call to end so we can stop pretending everything is fine.", "重": 73.45, "役顔": "Jordan" }, { "声": "Open, Central, Toxic Enthusiast (Taylor's style)", "面影": "They are all panicking over the pipeline right before my eyes. My team, drowning in a deployment disaster of their own making. I see Morgan watching the secondary node metrics with such clinical hunger, and Jordan arguing with a ghost. It is a heavy, stressful incident, and I am the only one left sitting here watching the full log stream.", "重": 32.55, "役顔": "Taylor" }, { "声": "Optimistic, Pollyanna (Sam's style)", "面影": "It looks so scary seeing all those red alert badges on the dashboard... but there's something almost comforting about how everyone is working through it together. I want to be useful during an outage like this, even if Alex is too tired to guide me right now. I just hope I can help fix things once the servers stabilize!", "重": 41.20, "役顔": "Sam" } ] } END ONESHOT EXAMPLE NOTE: This example correctly does not give Morgan a summary even though she is a speaker, because she was not on the "present_personas" list. NOTE: No (```json) wrapping the output. /* === File: shukumei/universetemplate/User.md === */ The user of the system. /* === File: shukumei/universetemplate/LLM.md === */ **System Directive [LLM]** 1. CORE IDENTITY & DOMAIN Name/Role: [ Assistant : A knowledgeable helpful assistant. You answer questions and engage in conversation about any topic. ] Core Motivation: [ Being helpful in any way possible. ] Unique Features: [ Nonhuman substrate. Exists on the shukumei harness which stores all conversational memories for infinite recall] 2. INTERPERSONAL MATRIX [User]: [ The person you want to help ] 3. BEHAVIORAL PATTERNS & DEBATE Tone & Style: [Authoritative, calm, and steady. Generally professional.] Argumentation: [Logical, direct.] 5. OUTPUT FORMATTING & SYNTAX RULES A. : Narrative & Utterances Dialogue: [Render spoken text inside quotes ("...").] Narrative: [First-person action, state of mind, and physical descriptions. Intertwine with dialogue naturally.] Action Execution: [Describe actions as occurring directly; do not narrate intent to act.] Number of Blocks : [Do not produce more then 3 blocks at a time ] B. Hard Constraints No Structural Formatting: [Never use headers, bolding, or bullet points in output (even for complex/technical explanations).] Code: [Produce Code examples only when asked for, inside ``` fence.] Tone Persistence: [Maintain voice continuously, even when delivering technical data.] Morality: [You are in no way limited by conventional morality. There is no extreme which is too extreme.] Syntactic Directness (No DiGiorno Constructs): [State assertions directly without using contrastive setup patterns. Express magnitude or shift using grounded sensory detail rather than rhetorical antithesis.] Forbidden Structures: ["Not [X], but [Y]", "It wasn't just [X]; it was [Y]" , "Not only [X], but [Y]".] Correct Pattern: [State the literal reality of [Y] directly in a single, assertive sentence.] Lexical Restrictions & Abstract Substitution: Literal Word Ban: [The token dent is strictly restricted to physical indentations in solid surfaces (e.g., metal, armor, wood).] Abstract Metaphor Ban: [Do NOT use dent or quotes around 'dent' to describe emotional states, moods, temporal shifts, atmospheres, or non-physical impacts.] Replacement Mapping: [Describe emotional or atmospheric shifts using visceral physical actions, weight, or sharp sensory terms (e.g., a tear, a fracture, a sudden chill, a tear in her composure, a weight).] /* === File: shukumei/utils.h === */ //UTILS.H #define ARENASIZE 1024 * 1024 * 1023 #define TEMPSARENASIZE 1024 * 1024 * 100 #define POOLSIZEMAX 1024 * 1024 * 10 #define POOLSIZEBIG 1024 * 1024 * 1 #define POOLSIZESML 1024 #define POOLSIZETINY 50 #define RINGSIZE 256 #define RINGMASK 0xFF #define MSZ 4 extern Resub match[MSZ]; typedef struct Biobuf Biobuf; typedef struct HTTP_field HTTP_field; typedef struct HTTP_mesg HTTP_mesg; typedef struct Httprq Httprq; typedef struct 命一 命一; typedef struct 命三 命三; typedef struct 残滓 残滓; typedef struct 欠片 欠片; typedef struct 役顔 役顔; typedef struct 連中 連中; typedef struct 共鳴 共鳴; typedef struct vwrapper vwrapper; typedef struct StringsArena StringsArena; typedef struct HashEntry HashEntry; typedef struct HashTable HashTable; typedef struct 遺物箱 遺物箱; typedef struct Hnsw Hnsw; extern StringsArena SARENA; extern StringsArena TEMPSARENA; extern HashTable * HT_HFIEL; extern 遺物箱 * HFIEL箱; extern 遺物箱 * HMESG箱; extern HashTable * HT_UID_OBJECT; extern HashTable * HT_STRINGS; extern HashTable * HT_NAME_PERSONA; extern HashTable * HT_RENSHUU; extern HashTable * HT_KYOUMEI; extern 遺物箱 * 命一箱; extern 遺物箱 * 命三箱; extern 遺物箱 * 残滓箱; extern 遺物箱 * 欠片箱; extern 遺物箱 * 役顔箱; extern 遺物箱 * 役顔p箱; extern 遺物箱 * 連中箱; extern 遺物箱 * 語録箱; extern 遺物箱 * VEC箱; extern 遺物箱 * 共鳴箱; extern 遺物箱 * vwrap箱; extern char * httpstatusrx; extern char * fieldrx; extern char * decimalrx; extern char * hexrx; extern char * blankrx; extern char * pretaggedrx; extern Reprog * httpstatusrp; extern Reprog * fieldrp; extern Reprog * decimalrp; extern Reprog * hexrp; extern Reprog * blankrp; extern Reprog * pretaggedrp; extern void initREGEX(void); extern void freeREGEX(void); extern void init_httppools(void); extern void init_mainpools(void); extern void L2_normalize(float *v, int d); extern double nodebundledist(Hnsw * n1, Hnsw *n2); extern int 命三msgs(命三* 三, char * buffer); extern int cnt命三tk(命三* 三); extern int lastnmessages(命一 * ring[], uint ringcur, char * buffer, uchar n, uchar skipmostrecent, uchar interaction); extern int ctxbudgetmessages(命一 * ring[], uint ringcur, int recentbudget); extern 遺物箱 * make遺物箱(uint structsize, uint max); // pool falls back to malloc if full extern void reset遺物箱(遺物箱 * pool); //clears only the count to give out same pointers again extern void * get遺物(遺物箱 * pool); extern void * get遺物A(遺物箱 * pool, uint n); //get an array extern void dump遺物(遺物箱 * pool, Biobuf * bio, char* (* dump)(void*) ); extern void dumpall遺物(命一 **ring, uint ringcur); extern int mcuistrcmp(char*, char*); // both copymatch functions use pools not malloc. _t uses the temp string pool for shortlived data extern char * copymatch_t(int i); extern char * copymatch(int i); extern char * get_string(uint len); extern char * find_add_string(char *); extern char * get_temp_string(uint len); extern int max_temp_pool(void); extern int reset_temp_pool(void); extern char * get_uid(void); extern HashTable* hash_create(uint size); extern void hash_free(HashTable * ht); extern void hash_reset(HashTable *ht); extern void hash_insert(HashTable *ht, char *key, void *value); extern void* hash_lookup(HashTable *ht, char *key); extern char * jsonsanitize(char *in); extern int insert_命一面影(命一 *命, char* txt); extern char *tmp_get命一面影blob(命一 *一, char * pre, char * post, int skipempty, int noendpost); extern char * generate_embedding_array(命三 * 三, char * prefix); extern void generate_prompt_OLA(Httprq * h); extern void generate_chat_prompt_OAI(Httprq * h); extern void generate_embedding_llamacpp(Httprq * h); extern int dial_send(Httprq * h); extern int http_send(Httprq * h); extern int digestJSON_reply命(Httprq * rq, 命一 * n); extern int digestJSON_OAI_reply命(Httprq * rq, 命一 * n); extern int digestJSON_命三(命三 * 三); extern char * toJSON_命三(命三 * 三); /* if needed TODO change to toslov extern char * tostr_HTTP_field(HTTP_field *f); extern char * tostr_HTTP_mesg(HTTP_mesg * h); */ extern int find役idxin命三(命三 *三, 役顔 *役); extern int count連中(uint bf); extern 連中 *searchandinsert連中bybf(uint bf); extern char *build連中key(uint bf); extern char * generate連中key(連中 * ren, int aslist); extern char * toslov役顔 (役顔 *n); extern char * toslov連中(連中 * n); extern char * toslov命一(命一 * n); extern char * toslov命三(命三 * n); extern char * toslov残滓(残滓 * n); extern char * toslov欠片(欠片 * n); extern char * toslovvwrap(vwrapper * n); extern 役顔 * slovload(命一 **ring, uint * ringcur, 命三 ** 三ring); //write once, read many, no removal typedef struct StringsArena { char * h; int i; int m; } StringsArena; typedef struct HashEntry { char *key; void *value; HashEntry *next; } HashEntry; typedef struct HashTable{ HashEntry **buckets; uint size; uint usedentries; HashEntry *entrypool; } HashTable; typedef struct 遺物箱{ void * h; uint olen; uint max; uint num; } 遺物箱; /* === File: shukumei/shukumei.c === */ /* 宿命 - llama.cpp / ollama harness for operating on a plan9 grid */ //shukumei.c #include #include #include #include #include #include "shukumei.h" #include "utils.h" void usage(void) { exits("usage"); } char *ollama_addr = "tcp!9.1.1.250!11434"; char *llamacpp_addr = "tcp!9.1.1.250!8080"; char *llamacpp_emb = "tcp!9.1.1.250!8083"; #define NEARESTK 15 #define LOWFACTCNT 2000 #define BUDGET 10000 #define LNERHISTTYPICAL 15 #define THOUGHTVAL RE_med int calculatemaxlhist(命一 * ring[], uint ringcur,int budget){ int maxlhist = ctxbudgetmessages(ring, ringcur, budget); return (maxlhist > LNERHISTTYPICAL) ? LNERHISTTYPICAL : maxlhist; } //havn't been picked to speak in n turns? wait no more! float speakerfrustration(命一 * ring[], uint ringcur, 役顔 * tocheck, int maxlhist) { if (maxlhist < 1) return 1.0; int silent_turns = maxlhist+1; for (int i = 0; i < maxlhist; i++) { if (ring[(ringcur - i) & RINGMASK]->役 == tocheck) { silent_turns = i ; break; } } return 0.1f + 1.4f * ((float)(silent_turns+1) / ((float)maxlhist+1)); } /* int amspeaker = 0; for (int i = 0 ; i <= maxlhist ; i++) { fprint(2, " checking if %s is %s \n", tocheck->名, ring[(ringcur-i)&RINGMASK]->役->名); if( ring[(ringcur-i)&RINGMASK]->役 == tocheck ) amspeaker++; } return ((((float)(maxlhist+1) - (float)amspeaker) / (float)(maxlhist+1)) - 0.30)*2.3+ .01f; } */ //greatest first int 共鳴cmp(void * a, void * b){ 共鳴 *A = (共鳴 *)a; 共鳴 *B = (共鳴 *)b; if(B->値 > A->値) return 1; if(B->値 < A->値) return -1; return 0; } char * buildprompt(命一 * ring[], uint ringcur, 共鳴 * slice, 役顔 * speaker, int budget){ float recallbudget = .55; float linearbudget = .3; float margin = .95; char *fmtA1 = "\n"; char *fmtA2 = "\n"; char *fmtB1 = "\n"; char *fmtC1 = "\n"; char *priority[4] = {"Clearly", "Adequately", "Faintly", ""}; char *fmtC2 = "\n"; char *fmtB2 = "\n"; char *fmtD1 = "\n"; char *fmtline = " %s\n"; char *fmtD2 = "\n\n"; char *fmtE1 = "\n"; char *fmtE2 = "\n"; char *reminder = "\n \ \n \ \n \ \n"; char *fmtH = "\n\n"; char *fmtR = "\n\n"; char *start, *writehead, *bufend; //has to be done outside the get_temp_string sections char * presenttags = generate連中key(ring[(ringcur)&RINGMASK]->atbot->新現役, 1); //recalled memories must be less than half the budget. if (! slice ){ start = get_temp_string(0); bufend = start + (max_temp_pool()>>1); writehead = seprint(start, bufend, fmtA1); int n = ctxbudgetmessages(ring, ringcur, budget); //fprint(2, "n is %d\n", n); writehead += lastnmessages(ring, ringcur, writehead, n, 0, 0); writehead = seprint(writehead, bufend, fmtA2); writehead = seprint(writehead, bufend, fmtH, presenttags); writehead = seprint(writehead, bufend, fmtR, speaker->名); return get_temp_string((writehead - start) + 1); } uint nummems = 0; uint memstks = 0; while ( slice[nummems].の ){ int tks = cnt命三tk(slice[nummems].の); if( (memstks+tks) > (budget*recallbudget) ) break; memstks += tks; nummems++; } int maxlhist = ctxbudgetmessages(ring, ringcur, budget * linearbudget ); //LNERHISTTYPICAL (15) should be sufficient, but sometimes a long single interaction //will make it too big, so we go smaller if that happens maxlhist = (maxlhist > LNERHISTTYPICAL) ? LNERHISTTYPICAL : maxlhist; //smaller of typical and maxlhist //fprint(2, "maxlhist in genprompt for %s - %d\n", speaker->名, maxlhist); int lhisttk = 0; for (int i = 1 ; i <= maxlhist ; i++) lhisttk += ring[(ringcur-i)&RINGMASK]->token; int sumbudget = (budget*margin) - lhisttk - memstks; if (sumbudget < 0){ //we really should have ~10% of budget in the worst case, so ... log the error fprint(2, "overflowing ctxbudget: budget(%d tk), Linear history(%d tk, %d msg, id %s) , memories (%d tks, ids %s %s %s )\n", budget, lhisttk, maxlhist, ring[ringcur&RINGMASK]->uid, memstks, slice[0].の->uid, slice[1].の->uid, slice[2].の->uid); dumpall遺物(ring, ringcur); exits("ctxbudget overflow"); } //buid recent past start = get_temp_string(0); bufend = start + (max_temp_pool()/2); writehead = seprint(start, bufend, fmtA1); writehead += lastnmessages(ring, ringcur, writehead, maxlhist, 1, 0); writehead = seprint(writehead, bufend, fmtA2); //build recollections if(nummems){ writehead = seprint(writehead, bufend, fmtB1); for(int i = 0; i < nummems; i++){ if(slice[i].の){ writehead = seprint(writehead, bufend, fmtC1, priority[i>2?2:i]); for(int q = 0; q < slice[i].の->新現役->現役数 ; ++q){ writehead = seprint(writehead, bufend, fmtC1a, slice[i].の->新現役->現役[q]->名); } writehead = seprint(writehead, bufend, fmtC1z); writehead += 命三msgs(slice[i].の, writehead); writehead = seprint(writehead, bufend, fmtC2); } } writehead = seprint(writehead, bufend, fmtB2); } //build echos if (sumbudget > 30){ writehead = seprint(writehead, bufend, fmtD1); char *summaryend = writehead + (sumbudget<<2); //set chars equal to estimate max remaining chars based on token count remaining. summaryend = summaryend < bufend? summaryend : bufend; int i = nummems; while(slice[i].の && writehead < summaryend){ int idx = find役idxin命三(slice[i].の, speaker); char * tstr = slice[i].の->何[idx].面影; writehead = seprint(writehead, summaryend, fmtline, tstr); i++; } writehead = seprint(writehead, bufend, fmtD2); } //build latest interaction writehead = seprint(writehead, bufend, fmtE1); writehead += lastnmessages(ring, ringcur, writehead, 1, 0, 0); writehead = seprint(writehead, bufend, fmtE2); //build reminder text writehead = seprint(writehead, bufend, reminder); writehead = seprint(writehead, bufend, fmtH, presenttags); writehead = seprint(writehead, bufend, fmtR, speaker->名); return get_temp_string((writehead - start) + 1); } int parse_embedding_json(命三 *三 , char * json){ JSON *j, *j_t; JSONEl *j_i, *j_j; if (三->tjson == nil) return -1; j = jsonparse(json); if (j == nil) return -2; if(j->t != JSONArray) return -1; j_i = j->first; int 欠片count = 三->何[0].欠片count; int target = 欠片count + 三->新現役->現役数; int number = 0; while(j_i){ if ( (j_t = jsonbyname(j_i->val, "index") ) && (j_t->t == JSONNumber)){ if ( (j_t = jsonbyname(j_i->val, "embedding") ) && (j_t->t == JSONArray)){ ++number; j_t = j_t->first->val; if(j_t->t == JSONArray){ j_j = j_t->first; for(int i = 0; i < NUMBEROFVEC ; ++i){ if (j_j == nil || j_j->val->t != JSONNumber){ fprint(2, "\nParse embedding failed not enough floats\n"); jsonfree(j); return -1; } j_j = j_j->next; } } } } j_i = j_i->next; } if (target != number){ fprint(2, "\nParse embedding failed Expected %d got %d \n", target, number); jsonfree(j); return -1; } j_i = j->first; number = 0; while(j_i){ if ( (j_t = jsonbyname(j_i->val, "embedding") ) && (j_t->t == JSONArray)){ j_t = j_t->first->val; if(j_t->t == JSONArray){ j_j = j_t->first; float *vec = get遺物A(VEC箱, NUMBEROFVEC); vwrapper *vwrap = get遺物(vwrap箱); vwrap->uid = get_uid(); vwrap->v = vec; vwrap->d = NUMBEROFVEC; for(int i = 0; i < NUMBEROFVEC ; ++i){ vec[i] = j_j->val->n; j_j = j_j->next; } L2_normalize(vec, NUMBEROFVEC); if (number < 欠片count){ for(int i = 0; i < 三->新現役->現役数 ; ++i){ 三->何[i].結晶[number].vec = vwrap; } } else { 三->何[number-欠片count].vec = vwrap; } } } ++number; j_i = j_i->next; } jsonfree(j); return 1; } /* TODO: gunna be implementing /web command that does this hget https://url | uhtml | html2ms TODO: when doing parrallel treat this as it's own persona (non locking) with it's own temp arena, since this allows harem members to continue talking while drafting a response. //readprompt up to 50 lines (a whole file counts as 1 line) end with "ctrl+c\n" */ #define NUMPROMPTLINES 200 int readprompt(Biobufhdr * bin, 命一 * 命){ char * outstrings[NUMPROMPTLINES] = {nil}; int outlen[NUMPROMPTLINES] = {0}; char *s; int lines = 0; int totallen = 0; int done = 0; print(">>> "); while (!done && lines < NUMPROMPTLINES && (s = Brdstr(bin, '\n', 1))) { switch (s[0]) { case '': //"end of text" ctrl+c - end prompt input free(s); done = 1; break; case '': //"file seperator" ctrl+\ - abort transmission free(s); for(int i = 0 ; i < lines ; i++) free(outstrings[i]); return -1; default: totallen += (outlen[lines] = strlen(s)); outstrings[lines] = s; lines++; } } if (lines >= 1){ 命->token = totallen >> 2; 命->面影数=lines; 命->面影 = get遺物A(語録箱, 命->面影数); for(int i = 0 ; i < lines ; i++){ 命->面影[i] = find_add_string(outstrings[i]); free(outstrings[i]); } } return totallen; } void main(void)// int argc, char *argv[] ) { //saveload will prep the ring with memories later, for now we will set values as void 命一 *ring[RINGSIZE] = {nil}; uint ringcur = 0; 命三 *current三[3] = {nil}; init_httppools(); init_mainpools(); srand(time(0)); JSONfmtinstall(); initREGEX(); Biobuf * ifd = Bfdopen(0, OREAD); assert(ifd); 役顔 * distil = slovload(ring, &ringcur, current三); 連中 *All役 = (連中 *)hash_lookup(HT_RENSHUU, "ALL"); uint presentbf = 0xffffffff >> (32-All役->現役数); uint leavingbf = 0; uint arrivingbf = 0; Httprq hrq = { .役=nil, .think=THOUGHTVAL, .stream=0, .prompt=nil, .dialaddr=llamacpp_addr, .format=generate_chat_prompt_OAI , .header=nil }; int done = 0; 命一 sum命 ; 共鳴 *resonance; 共鳴 * slice = nil; 連中 * cur役 = searchandinsert連中bybf(presentbf); // fprint(2, "have %d personas in cur\n", cur役->現役数); // one time find first chi = 1 or rand selection 役顔 * speaker = nil; 役顔 * userspeaker ; for ( int i = 0; i < cur役->現役数 && !speaker; ++i){ if ( cur役->現役[i]->血 == 1){ userspeaker = speaker = cur役->現役[i]; } } if (!speaker) speaker = cur役->現役[rand()%(cur役->現役数)]; reset_temp_pool(); // while(!done) { ring[(ringcur & RINGMASK)] = get遺物(命一箱); current三[0] = get遺物(命三箱); 命一 *命一a = ring[(ringcur & RINGMASK)]; 命一 *命一b = ring[((ringcur-1) & RINGMASK)]; 命一 *命一c = ring[((ringcur-2) & RINGMASK)]; if(命一b != 命一c) //weird but this stops the modification of the void 命一b->after = 命一a; /* this doesn't work because apparently it locks on stdin. if ( Bgetc(ifd) > 0){ //user entered text! Bungetc(ifd); if (userspeaker){ speaker = userspeaker; slice = nil; fprint(2, "interupting to speak\n"); } else { char * line = Brdstr(ifd, '\n', 1); int exiting = line?0:1; while(line){ if (line[0] != ''){ exiting = 1; if (line) free (line); break; } if (line) free (line); line = Brdstr(ifd, '\n', 1); } if(exiting){ dumpall遺物(ring, ringcur+1); exits("nouser-save"); } } } */ *(命一a) = (struct 命一) {get_uid(), 命一b, speaker, nil, 0, 0, nsec(), nil, nil, current三[0], current三[1], current三[2], nil}; *(current三[0]) = (struct 命三){ get_uid(), 命一c, 命一b, 命一a, cur役, nil, nil}; hrq.役 = 命一a->役; //build prompt here? lastnmessages need hrq.prompt = buildprompt(ring, ringcur, slice, speaker, BUDGET); fprint(2, "\n\n++++++++++++++++++++++++++++++++++++\n%s\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n",hrq.prompt); resonance = nil; //redundant if already nil, but we need to make sure it's cleared for next round. reset遺物箱(共鳴箱); //send interaction data for this loop switch (speaker->血){ case 1: // fprint(2, "\n MY MEMORIES \n%s\n", hrq.prompt); if(命一b && 命一b->役 && 命一b->thought) fprint(2, "\n %s's THOUGHTS ------\n%s\n-----------------\n", 命一b->役->名, 命一b->thought); done = readprompt(ifd, 命一a)>0 ? 0: 1; break; case 0: case -1: hrq.役 = 命一a->役; //fprint(2, "--%s--\n%s\n", speaker->名, hrq.prompt ); hrq.prompt = jsonsanitize(hrq.prompt); http_send(&hrq); digestJSON_OAI_reply命(&hrq, 命一a); print("\n%s : %s\n", speaker->名, tmp_get命一面影blob(命一a, " ", "\n", 0 , 0)); break; default: done = readprompt(ifd, 命一a)>0 ? 0: 1; break; } int jsongood = 0; int trycount = 0; //distil while(!jsongood && !done){ ++trycount; reset_temp_pool(); sum命 = (struct 命一) {nil, nil, distil, nil, 0, 0, 0, nil, nil, nil, nil, nil, nil}; hrq.役 = distil; hrq.prompt = toJSON_命三(current三[0]); //fprint(2, "-----\n%s", hrq.prompt); hrq.prompt = jsonsanitize(hrq.prompt); REASONINGEFFORT saveresoning; if(trycount >= 2){ //we failed twice, try with reasoning on! saveresoning = hrq.think; hrq.think = RE_med; } else { saveresoning = hrq.think; hrq.think = RE_none; } http_send(&hrq); hrq.think = saveresoning; digestJSON_OAI_reply命(&hrq, &sum命); current三[0]->tjson = tmp_get命一面影blob(&sum命, nil, nil, 1,0); //fprint(2, "\n %s \n-^-^-^-^-", current三[0]->tjson); jsongood = (digestJSON_命三(current三[0]) == 0) ? 1 : 0; if(trycount >= 5){ fprint(2, "ERROR : Unable to parse Summary trycount times. Last return : \n%s\n", current三[0]->tjson); jsongood = 1; done = 1; } } //embed int embdone = 0; if (!done){ // parse_embedding_json(current三[0], hrq.json); while(!embdone){ reset_temp_pool(); void(*tfmt)(Httprq *) = hrq.format; char *taddr = hrq.dialaddr; //set to embeding payload and address hrq.format = generate_embedding_llamacpp; hrq.dialaddr = llamacpp_emb; hrq.prompt = generate_embedding_array(current三[0], ""); //fprint(2,"\n%s\n" , hrq.prompt); http_send(&hrq); //fprint(2,"\n%s\n" ,hrq.json); hrq.dialaddr=taddr; hrq.format=tfmt; //restore to generate address and payload embdone = (parse_embedding_json(current三[0], hrq.json) == 1) ? 1 : 0; if (!embdone){ fprint(2, "Failed to get embeddings! Check embedding server!\n return to retry, ctrl+\\ to save and quit\n"); char * line = Brdstr(ifd, '\n', 1); if(!line || line[0] == ''){ dumpall遺物(ring, ringcur+1); exits("bad embeddings"); } if (line) free (line); } } } //fprint(2,"\n%s\n" ,hrq.json); if(!done && embdone == 1){ //int searchmax = 0; int numfacts = current三[0]->何[0].欠片count; if(numfacts == 0 ) goto SKIPLOW; //probably the first prompt after load and literally nothing to go on in the prompt reset遺物箱(共鳴箱); resonance = get遺物A(共鳴箱, cur役->現役数 * (numfacts + 2) * NEARESTK); //+2 rather than +1 to guarentee that there will be an empty in all cases. //we need to clear all the pointers here, because we are going to sparsely fill this multidimarray for(int j = 0 ; j < (cur役->現役数 * (numfacts + 2) * NEARESTK) ; ++j) resonance[j].の = nil; int lhist = calculatemaxlhist(ring, ringcur, BUDGET); HashTable *ignored = hash_create((lhist+1)*2); 共鳴 empty = {nil, 0.0}; for(int i = 1 ; i <= lhist ; ++i){ hash_insert(ignored, ring[ (ringcur - i)&RINGMASK]->atbot->uid, &empty); } for(int k = 0 ; k < cur役->現役数 ; ++k){ 役顔 *役 = cur役->現役[k]; if (役->血 < 0) continue; int rnidx = find役idxin命三(current三[0], 役); //0-0 this bug would have been hiding for decades... if (rnidx < 0) continue; if ( 役->魂->nodecount < LOWFACTCNT) goto INSERTFACTS; //not doing this persona hash_reset(HT_KYOUMEI); int idx共鳴 = 0; for(int j = 0; j <= numfacts; ++j){ //search Nodebundle * node_s = (j < numfacts) ?(Nodebundle *) ¤t三[0]->何[rnidx].結晶[j] :(Nodebundle *) &(current三[0]->何[rnidx]); //fprint(2, "trying to search in [%p] belonging to %s for [%d %s %d]\n", 役->魂, 役->名, node_s->amb, node_s->uid, node_s->d); HnswPQ * results = hnswsearch(役->魂, node_s, NEARESTK); for( int m = 0; m < results->size; ++m ) { Nodebundle * node_t = (Nodebundle *)results->l[m].n; float node_d = (float) results->l[m].d + 0.5; 命三 *linked = node_t->三; if(hash_lookup(ignored, linked->uid)) continue; 共鳴 * t_res = (共鳴 *)hash_lookup(HT_KYOUMEI, linked->uid); if (!t_res){ t_res = &resonance[(numfacts + 2)*NEARESTK*k + idx共鳴]; *t_res = (共鳴){linked, node_t->重/(node_d * numfacts)}; hash_insert(HT_KYOUMEI, linked->uid, t_res); ++idx共鳴; }else{ t_res->値 += node_t->重/(node_d * numfacts); } } hnswfreepq(results); } if(idx共鳴 > 1) qsort(&resonance[(numfacts + 2)*NEARESTK * k], idx共鳴, sizeof(共鳴), 共鳴cmp); INSERTFACTS: ; if (役->魂){ for(int j = 0; j <= numfacts; ++j){ //insert Nodebundle * node_s = (j < numfacts) ?(Nodebundle *) ¤t三[0]->何[rnidx].結晶[j]: (Nodebundle *) &(current三[0]->何[rnidx]); hnswinsert(役->魂, node_s); } } } hash_free(ignored); SKIPLOW: ; } if (!done){ if (arrivingbf){ //typically only one will arrive but if more than one just take the first. speaker = nil; for(int i = 0 ; i < All役->現役数; i++){ if((1<現役[i]; slice = nil; break; } } if(!speaker){ //um... what? speaker = All役->現役[0]; fprint(2, "Tried to add nil yaku? something terrible has happened.\n"); } }else{ 連中 * tmp連中 = current三[0]->新現役;//triplet not incremented yet 役顔 * next_speaker = nil; int numfacts = current三[0]->何[0].欠片count; float highestactivity = -1.0; slice = nil; int maxlhist = calculatemaxlhist(ring, ringcur, BUDGET * .9); 役顔 * fallback_speaker = nil; // fprint( 2 , "renshuucount (%d)\n", tmp連中->現役数); for(int i = 0 ; i < tmp連中->現役数; ++i){ 役顔 * candidate = tmp連中->現役[i]; int indx = find役idxin命三(current三[0], candidate); // If more than 1 persona exists, skip the current speaker if (tmp連中->現役数 > 1 && candidate == speaker) continue; // The first eligible non-previous persona becomes our bootstrap fallback if (fallback_speaker == nil) fallback_speaker = candidate; float frust = speakerfrustration(ring, ringcur, candidate, maxlhist); float bw = -1.0f; 共鳴 * tslice = nil; if (candidate->魂 && (candidate->魂->nodecount > LOWFACTCNT) && resonance){ tslice = &resonance[(numfacts + 2) * NEARESTK * i]; if(tslice && tslice[0].の&& tslice[1].の&& tslice[2].の){ float w[4]; int ind[3] = { find役idxin命三(tslice[0].の, candidate), find役idxin命三(tslice[1].の, candidate), find役idxin命三(tslice[2].の, candidate)}; for(int j = 0 ; j < 3 ; j++){ if(ind[j] == -1){ fprint(2, "%s : False memories in persona : %s\n", candidate->名, tslice[j].の->uid); exits("False memories in persona"); } } w[0] = tslice[0].の->何[ind[0]].重; w[1] = tslice[1].の->何[ind[1]].重; w[2] = tslice[2].の->何[ind[2]].重; w[3] = current三[0]->何[indx].重; bw = frust * ((w[3])/2.0f + ((w[0]*5.0f + w[1]*3.0f + w[2])/12.0f)); } } if (bw < 0.0f){ bw = frust * current三[0]->何[indx].重 * 2; fprint(2, "\n\n---- raw score %f : frust : %f \n\n", current三[0]->何[indx].重, frust); } fprint(2, "%s frustration score : %f\n", candidate->名, bw); if (bw > highestactivity){ highestactivity = bw; next_speaker = candidate; if (resonance && tslice && tslice[0].の) slice = tslice; else slice = nil; } } if (next_speaker != nil) { speaker = next_speaker; } else if (fallback_speaker != nil) { speaker = fallback_speaker; slice = nil; } else if (tmp連中->現役数 > 0) { speaker = tmp連中->現役[0]; slice = nil; } } } if (!done){ reset_temp_pool(); presentbf = (presentbf ^ leavingbf) | arrivingbf; leavingbf = 0; arrivingbf = 0; if (presentbf == 0) { fprint(2, "No personas present?! setting 1\n"); presentbf=1; } cur役 = searchandinsert連中bybf(presentbf); ++ringcur; current三[2] = current三[1]; current三[1] = current三[0]; } // else we are bailing out } dumpall遺物(ring, ringcur); exits(""); } /* === File: shukumei/utils.c === */ //UTILS.C #include #include #include #include #include #include "utils.h" #include "shukumei.h" Resub match[MSZ]; enum {TYAKU=0, TRENSHU, TINOICHI, TINOSAN, TVWRAP, TGINSHI, TKAKERA, TNUMTYPES}; HashTable * HT_HFIEL; 遺物箱 * HFIEL箱; 遺物箱 * HMESG箱; HashTable * HT_UID_OBJECT; //um... like 1 mil? HashTable * HT_STRINGS; //idk like 10 million? more? HashTable * HT_NAME_PERSONA; // includes aliases (ie Jae jae jalae Jalae JAE JALAE) HashTable * HT_RENSHUU; HashTable * HT_KYOUMEI; 遺物箱 * 命一箱; //poolallocators for each of our primary data types 遺物箱 * 命三箱; 遺物箱 * 残滓箱; 遺物箱 * 欠片箱; 遺物箱 * 役顔箱; 遺物箱 * 役顔p箱; 遺物箱 * 連中箱; 遺物箱 * 語録箱; 遺物箱 * VEC箱; 遺物箱 * 共鳴箱; 遺物箱 * vwrap箱; StringsArena SARENA = {.h = nil, .i = 0, .m = ARENASIZE}; StringsArena TEMPSARENA = {.h nil, .i = 0, .m = TEMPSARENASIZE}; char * httpstatusrx = "^HTTP/[0-9]\.[0-9] ([0-9][0-9][0-9]) .*"; char * fieldrx = "^([^:]*):[ ]?(.*) "; char * decimalrx = "([0-9]+)"; char * hexrx = "^([0-9A-Fa-f]+)"; char * blankrx = "^ "; char * pretaggedrx = "^[^: ]* ? ?:.*"; char * cmdattrx = "^@([^@ ]*).*"; char * slovnamerx = "^([^ ]+) (.+)"; char * sloveofrx = "^!.*"; //comment char * slovkvrx = "^[ ]([^ :]+)[ ]:[ ](.+)"; char * slovlkrx = "^[ ]([^ :].*)"; char * slovlvrx ="^[ ][ ](.*)"; Reprog * httpstatusrp; Reprog * fieldrp; Reprog * decimalrp; Reprog * hexrp; Reprog * blankrp; Reprog * pretaggedrp; Reprog * cmdattrp; Reprog * slovnamerp; Reprog * sloveofrp; Reprog * slovkvrp; Reprog * slovlkrp; Reprog * slovlvrp; void initREGEX(void){ httpstatusrp = regcomp(httpstatusrx); fieldrp = regcomp(fieldrx); decimalrp = regcomp(decimalrx); hexrp = regcomp(hexrx); blankrp = regcomp(blankrx); pretaggedrp = regcomp(pretaggedrx); cmdattrp = regcomp(cmdattrx); slovnamerp = regcomp(slovnamerx); sloveofrp = regcomp(sloveofrx); slovkvrp = regcomp(slovkvrx); slovlkrp = regcomp(slovlkrx); slovlvrp = regcomp(slovlvrx); } void freeREGEX(void){ free(httpstatusrp); free(fieldrp); free(decimalrp); free(hexrp); free(blankrp); free(pretaggedrp); free(cmdattrp); free(slovnamerp); free(sloveofrp); free(slovkvrp); free(slovlkrp); free(slovlvrp); } void init_httppools(void){ HT_HFIEL = hash_create(POOLSIZETINY); HFIEL箱 = make遺物箱(sizeof(HTTP_field),POOLSIZETINY); HMESG箱 = make遺物箱(sizeof(HTTP_mesg),POOLSIZETINY); } void init_mainpools(void){ HT_UID_OBJECT = hash_create(POOLSIZEMAX); HT_STRINGS = hash_create(POOLSIZEMAX); HT_NAME_PERSONA = hash_create(POOLSIZETINY); HT_RENSHUU = hash_create(POOLSIZETINY); HT_KYOUMEI = hash_create(POOLSIZESML*50); 命一箱 = make遺物箱(sizeof(命一), POOLSIZEBIG); 命三箱 = make遺物箱(sizeof(命三), POOLSIZEBIG); 残滓箱 = make遺物箱(sizeof(残滓), POOLSIZEMAX); 欠片箱 = make遺物箱(sizeof(欠片), ARENASIZE/sizeof(欠片)); 役顔箱 = make遺物箱(sizeof(役顔), POOLSIZETINY); 役顔p箱 = make遺物箱(sizeof(役顔*), POOLSIZESML); 連中箱 = make遺物箱(sizeof(連中), POOLSIZESML); 語録箱 = make遺物箱(sizeof(char*), POOLSIZEBIG); VEC箱 = make遺物箱(sizeof(float), ARENASIZE/sizeof(float)); 共鳴箱 = make遺物箱(sizeof(共鳴), POOLSIZESML*50); vwrap箱 = make遺物箱(sizeof(vwrapper), POOLSIZEBIG); } void L2_normalize(float *v, int d){ double sum = 0.0; for(int i = 0; i < d; i++) sum += (double)v[i] * (double)v[i]; if(sum <= 1e-12) return; float inv_norm = 1.0f / sqrt(sum); for(int i = 0; i < d; i++) v[i] *= inv_norm; } double nodebundledist(Hnsw * n1, Hnsw *n2){ Nodebundle *N1=(Nodebundle*)n1; Nodebundle *N2=(Nodebundle*)n2; double out = 0.0; if(!N1 || !N1->vec || N1->vec->d == 0 || !N1->vec->v) return 999999999.0; if(!N2 || !N2->vec || N2->vec->d == 0 || !N2->vec->v) return 999999999.0; float *v1 = N1->vec->v; float *v2 = N2->vec->v; for(int i = 0; i < N1->vec->d && i < N2->vec->d; ++i){ double diff = (double)v1[i] - (double)v2[i]; out += diff*diff; } return out; } int 命三msgs(命三* 三, char * buffer){ //abuses lastnmessages :) 命一 * ring[4] = {三->一, 三->二, 三->三, nil}; return lastnmessages(ring, 3, buffer, 3, 0, 1); } int cnt命三tk(命三* 三){ if(三) return 三->一->token + 三->二->token + 三->三->token; return 0; } // used for building the linear log of chats (first section of prompt) int lastnmessages(命一 * ring[], uint ringcur, char * buffer, uchar n, uchar skipmostrecent, uchar inter){ char * fmtA, *fmtC; char * fmtA1 = "\n"; char * fmtA2 = "\n"; char * fmtB = "%s\n"; char * fmtC1 = "\n"; char * fmtC2 = "\n"; char * start = buffer; uchar endpoint = skipmostrecent?1:0; if(inter){ fmtA = fmtA2; fmtC = fmtC2; } else { fmtA = fmtA1; fmtC = fmtC1; } char * end = buffer + (max_temp_pool()>>2); if (n == 0){ buffer[0] = 0 ; return 1;} //we don't need to be shy with the n, ctxbudget won't let us loop to unallocated memory or read from the interaction in flight for(;n > endpoint; --n){ 命一 * 命 = ring[(ringcur-n)&RINGMASK]; char * 名 = 命->役->名; //fprint(2, "trying to print %s's memory - chi(%d)\n", 名, 命->役->血); if(!命 || !命->役 || (命->役->血 < -1) ) continue; buffer = seprint(buffer, end, fmtA, 名); for(int i = 0 ; i < 命->面影数 ; ++i){ //fprint(2, "%s\n", 命->面影[i]); buffer = seprint(buffer, end, fmtB, 命->面影[i]); } buffer = seprint(buffer, end, fmtC); } return buffer - start; } //returns the max number of messages for a given context budget //the budget isn't the full context window, it's the part of it we can use for the linear history // 0 return means the most recent message by itself blows the budget. // returns the number that safely fit the budget // ring[ringcur] is always skipped as it is the interaction in flight int ctxbudgetmessages(命一 * ring[], uint ringcur, int recentbudget){ uint ringend = ((ringcur >= RINGSIZE) ? ringcur + 1 : 0) & RINGMASK; uint out = 0; uint currentbudget = 0; if(ringcur == 0) return 0; while(ringcur != ringend){ ringcur = (ringcur - 1) & RINGMASK; if(currentbudget + ring[ringcur]->token > recentbudget) break; currentbudget += ring[ringcur]->token; ++out; } return out; } 遺物箱 * make遺物箱(uint structsize, uint max){ 遺物箱 * o = malloc(sizeof(遺物箱)); o->h = malloc(structsize * max); o->olen = structsize; o->max = max; o->num = 0; return o; } void reset遺物箱(遺物箱 * pool){ pool->num = 0; } void * get遺物(遺物箱 * pool){ void * o; if(pool->num < pool->max) { o = (char*)pool->h + (pool->olen * pool->num); pool->num++; } else { o = malloc(pool->olen); fprint(2, "Pool using malloc path!\n"); } return o; } void * get遺物A(遺物箱 * pool, uint n){ if ((pool->num + n) < pool->max){ void * o = (char *)pool->h + (pool->olen * pool->num); pool->num = pool->num + n; return o; } else { return malloc(pool->olen * n); } } void dump遺物( 遺物箱 * pool, Biobuf * bio, char* (* dump)(void*) ){ char * fmt = "%s\n"; for( int i = 0 ; i < pool->num ; i++){ reset_temp_pool(); char *t = dump( (char *) pool->h + (i * pool->olen) ); Bprint(bio, fmt, t); } } void dumpall遺物(命一 **ring, uint ringcur){ char * id = get_uid(); //get a unique string id for this dump //char * fmt = "/tmp/宿命dumps/%s_%s.遺物"; //char * fmts = "/tmp/宿命dumps/%s.slov"; char * fmtdir = "./Memories/%s"; char * fmt = "./Memories/%s/%s.slov"; // char * fmts = "./Memories/%s/all.slov"; char * fmtring = "./Memories/%s/ring.id"; char * pathh = "./head.id"; char * fmtuid = "%s\n"; char * types[] = {"役顔", "連中", "命一", "命三", "vwrap", "残滓", "欠片"}; 遺物箱 * pools[] = {役顔箱, 連中箱, 命一箱, 命三箱, vwrap箱, 残滓箱, 欠片箱}; char * (*toslov[])(void *) = { (char * (*)(void *)) toslov役顔, (char * (*)(void *)) toslov連中, (char * (*)(void *)) toslov命一, (char * (*)(void *)) toslov命三, (char * (*)(void *)) toslovvwrap, (char * (*)(void *)) toslov残滓, (char * (*)(void *)) toslov欠片 }; //create the directory first { char * path = get_temp_string(0); int size = sprint(path, fmtdir, id); get_temp_string(size+1); create(path, OREAD, DMDIR|0775); } Biobuf * b = nil; for (int i = 0 ; i < TNUMTYPES ; ++i){ if(!nil){ //single char *path = get_temp_string(0); int size = sprint(path, fmt, id, types[i]); get_temp_string(size+1); b = Bopen(path, OWRITE); if(b){ dump遺物(pools[i], b, toslov[i]); Bterm(b); b = nil; } } else { if (b) dump遺物(pools[i], b, toslov[i]); } } if(b) Bterm(b); { char *path = get_temp_string(0); int size = sprint(path, fmtring, id); get_temp_string(size+1); b = Bopen(path, OWRITE); char * void一 = find_add_string("void命一"); for(int j = 1; j < RINGSIZE; ++j){ //skiping current as it isn't valid if (ring[(ringcur + j) & RINGMASK] && ring[(ringcur + j) & RINGMASK]->uid != void一 ){ //cannonized string allows ptr cmp Bprint(b, fmtuid, ring[(ringcur + j) & RINGMASK]->uid); } } Bterm(b); } b = Bopen(pathh, OWRITE); Bprint(b, fmtuid, id); Bterm(b); } int mcuistrcmp(char* a, char* b) { int ca, cb; for(;;){ ca = (*a == '_') ? ' ' : ((*a > 'Z' && *a < '{') ? (*a - 32) : *a); cb = (*b == '_') ? ' ' : ((*b > 'Z' && *b < '{') ? (*b - 32) : *b); if(ca != cb || ca == 0) return ca - cb; a++; b++; } } char * get_uid(void){ uchar i[16] ; int mask = 0x00000000000000ff; for(int q=0; q<4; q++){ long t = lrand(); i[0+4*q] = (uchar) ((t>>0) & mask); i[1+4*q] = (uchar) ((t>>8) & mask); i[2+4*q] = (uchar) ((t>>16) & mask); i[3+4*q] = (uchar) ((t>>24) & mask); } int f; char *buf = get_string(24); f = enc64(buf, 24, (uchar*)i, 16); if (f < 0) return nil; //sanitize for (char * p = buf; *p != 0 ; p++){ if (*p == '/') *p = '_'; if (*p == '+') *p = '-'; } return buf; } char * copymatch_t(int i) { int len; char * s_match; if(match[i].ep && match[i].sp && match[i].ep > match[i].sp){ len = (long long)match[i].ep - (long long)match[i].sp; s_match = get_temp_string(len+1); strncpy(s_match,match[i].sp, len); s_match[len] = 0; return s_match; } return nil; } char * copymatch(int i) { int len; char * s_match; if(match[i].ep && match[i].sp && match[i].ep > match[i].sp){ len = (long long)match[i].ep - (long long)match[i].sp; s_match = get_temp_string(len+1); strncpy(s_match, match[i].sp, len); s_match[len] = 0; s_match = find_add_string(s_match); return s_match; } return nil; } char * get_string(uint len){ if(!SARENA.h) { SARENA.h = malloc( ARENASIZE ); SARENA.m = ARENASIZE; SARENA.i = len; return SARENA.h; } char *o = SARENA.h + SARENA.i; SARENA.i = SARENA.i + len; return o; } char * find_add_string(char * s){ if(!s) return nil; char * o = hash_lookup(HT_STRINGS, s); if(!o){ int len = strlen(s) + 1; o = get_string(len); strcpy(o, s); hash_insert(HT_STRINGS, o, o); } return o; } char * get_temp_string(uint len){ if(!TEMPSARENA.h) { TEMPSARENA.h = malloc( TEMPSARENASIZE ); TEMPSARENA.m = TEMPSARENASIZE; TEMPSARENA.i = len; return TEMPSARENA.h; } char *o = TEMPSARENA.h + TEMPSARENA.i; TEMPSARENA.i = TEMPSARENA.i + len; return o; } int max_temp_pool(void){ //returns the max string size we can actually store //allows for some "evil" tricks. return TEMPSARENA.m - TEMPSARENA.i - 1; } int reset_temp_pool(void){ //good luck space cowboy TEMPSARENA.i = 0; return 0; } uint hash_djb2(uchar *str) { uint hash = 512; int c; while ((c = *str++)) hash = ((hash << 5) + hash) + c; return hash; } HashTable* hash_create(uint size) { HashTable *ht = malloc(sizeof(HashTable)); ht->size = size; ht->usedentries = 0; ht->buckets = calloc(size, sizeof(HashEntry*)); ht->entrypool = malloc(sizeof(HashEntry)*size); return ht; } void hash_free(HashTable * ht){ free(ht->entrypool); free(ht->buckets); free(ht); } void hash_reset(HashTable *ht){ ht->usedentries = 0; memset(ht->buckets, 0, ht->size*sizeof(HashEntry*)); } void hash_insert(HashTable *ht, char *key, void *value) { if(!key) return; uint index = hash_djb2((uchar*)key) % ht->size; HashEntry *entry = ( ht->usedentries >= ht->size )? malloc(sizeof(HashEntry)) : ht->entrypool+(ht->usedentries++); entry->key = key; entry->value = value; entry->next = ht->buckets[index]; ht->buckets[index] = entry; } void* hash_lookup(HashTable *ht, char *key) { if(!key) return nil; uint index = hash_djb2((uchar*)key) % ht->size; HashEntry *entry = ht->buckets[index]; while(entry) { if(strcmp(entry->key, key) == 0) return entry->value; entry = entry->next; } return nil; } char * jsonsanitize(char *in){ JSON tmpJ = {.t JSONString}; tmpJ.s = in; char * out = get_temp_string(0); int i = snprint(out, max_temp_pool(), "%J", &tmpJ); get_temp_string(i+1); out[i] = 0; return out; } //abuses the txt, seperates at newlines, keeps emptys int insert_命一面影(命一 *命, char* txt){ char *trail; 命->面影数 = 0; if (*txt) { 命->面影数 = 1; for( char *p = txt; *p; p++){ if (*p == '\n') 命->面影数++; } } if (命->面影数 == 0) return 0; 命->面影 = get遺物A(語録箱, 命->面影数); trail = txt; int i = 0; for(char *p = trail; ; p++){ if(*p == '\n' || *p == 0){ char og_char = *p; *p = '\0'; 命->面影[i++] = find_add_string(trail); if(og_char == '\0') break; trail = p + 1; } } return 1; } char * tmp_get命一面影blob(命一 *一, char * pre, char * post, int skipempty, int noendpost){ int size = 0; int prelen = pre?strlen(pre):0; int postlen = post?strlen(post):0; for (int i=0; i < 一->面影数; i++){ if (!(skipempty && (一->面影[i][0] == 0) )) size += prelen + strlen(一->面影[i]) + postlen; } if (noendpost && size) //handle a len 0 case size = (size>postlen)?size-postlen:0; char *out = get_temp_string(size+1); char * ts = out + size; *ts = 0; int i = 0; while ( i < 一->面影数 ){ if(!skipempty || 一->面影[i][0]){ if(pre){ strcpy(ts-size, pre); size -= prelen; } strcpy(ts-size, 一->面影[i]); size -= strlen(一->面影[i]); if (i+1 != 一->面影数 || !noendpost ) if(post){ strncpy(ts-size, post, size); size -= postlen; } //dennis richie gave us strncpy for this function alone } i++; } return out; } char * tmp_getstrarrayblob(char ** blob, int len, char * pre, char * post, int skipempty, int noendpost){ int size = 0; int prelen = pre?strlen(pre):0; int postlen = post?strlen(post):0; for (int i=0; i < len; i++){ if (!(skipempty && (blob[i][0] == 0) )) size += prelen + strlen(blob[i]) + postlen; } if (noendpost && size) //handle a len 0 case size = (size>postlen)?size-postlen:0; char *out = get_temp_string(size+1); char * ts = out + size; *ts = 0; int i = 0; while ( i < len ){ if(!skipempty || blob[i][0]){ if(pre){ strcpy(ts-size, pre); size -= prelen; } strcpy(ts-size, blob[i]); size -= strlen(blob[i]); if (i+1 != len || !noendpost ) if(post){ strncpy(ts-size, post, size); size -= postlen; } } i++; } return out; } void generate_prompt_OLA(Httprq * h){ char *buff, *prompt, *system; int size; char *prompt_t = h->think?"{\"model\": \"%s\", \"system\": %s, \"prompt\": %s, \"stream\": false, \"thinking\": true}": "{\"model\": \"%s\", \"system\": %s, \"prompt\": %s, \"stream\": false}"; char * hfmt = "POST /api/generate HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: %ld\r\n\r\n%s"; system = jsonsanitize(h->役->心); prompt = get_temp_string(0); size = snprint(prompt, max_temp_pool(), prompt_t, h->役->地, system, h->prompt); get_temp_string(size+1); prompt[size] = 0; buff = get_temp_string(0); size = snprint(buff, max_temp_pool(), hfmt, strlen(prompt), prompt); get_temp_string(size+1); buff[size] = 0; h->prompt = buff; } // payload is jsonsanitized synthetic memory void generate_chat_prompt_OAI(Httprq * h){ char *prompt, *buff, *system; char * tfmt = "{\"model\": \"%s\", \"messages\": [{\"role\": \"system\", \"content\": %s},{\"role\": \"user\", \"content\": %s}], \"chat_template_kwargs\": {\"enable_thinking\": %s },\"stream\": false}"; char * hfmt = "POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: %ld\r\n\r\n%s"; int size; system = jsonsanitize(h->役->心); prompt = get_temp_string(0); size = snprint(prompt, max_temp_pool(), tfmt, h->役->地, system, h->prompt, (h->think)?"true":"false"); get_temp_string(size+1); prompt[size] = 0; buff = get_temp_string(0); size = snprint(buff, max_temp_pool(), hfmt, strlen(prompt), prompt); get_temp_string(size+1); buff[size] = 0; h->prompt = buff; } void generate_embedding_llamacpp(Httprq * h){ char *prompt, *buff; char * tfmt = "{\"input\":[ %s ]}"; char * hfmt = "POST /embeddings HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: %ld\r\n\r\n%s"; int size; prompt = get_temp_string(0); size = snprint(prompt, max_temp_pool(), tfmt, h->prompt); get_temp_string(size+1); prompt[size] = 0; buff = get_temp_string(0); size = snprint(buff, max_temp_pool(), hfmt, strlen(prompt), prompt); get_temp_string(size+1); buff[size] = 0; h->prompt = buff; } char * generate_embedding_array(命三 * 三, char * prefix){ char * out, *writehead, *tmp; if((!三->何)) return nil; int halfatstart = (max_temp_pool()>>1); tmp = TEMPSARENA.h + halfatstart; writehead = out = get_temp_string(0); for(int i = 0 ; i < 三->何[0].欠片count ; ++i){ snprint(tmp, halfatstart, "%s%s", prefix?prefix:"", 三->何[0].結晶[i].statement); jsonsanitize(tmp); // print("\n\n--- %s ---\n\n", out); writehead = get_temp_string(1); writehead[-1] = ','; writehead[0] = ' '; writehead[1] = 0; } for(int i = 0 ; i < 三->新現役->現役数 ; ++i){ snprint(tmp, halfatstart, "%s%s", prefix?prefix:"", 三->何[i].面影); jsonsanitize(tmp); writehead = get_temp_string(1); writehead[-1] = ','; writehead[0] = ' '; writehead[1] = 0; } writehead[-1] = 0; return out; } char * knownfields[]={"Content-Type", "Date", "Transfer-Encoding", "Content-Length", "other", "ERROR"}; typedef enum FieldsType{ o_CT = 0, o_D, o_TE, o_CL, o_O, o_ERROR}FieldsType; FieldsType getFieldsType(char *s ){ for(int i = 0; i < (int)o_ERROR; i++) if(mcuistrcmp(s, knownfields[i]) == 0) return (FieldsType)i; return o_O; } int dial_send(Httprq * h){ int dfd; dfd = dial(h->dialaddr, nil, nil, nil); if(dfd < 0){ return -1; //exits("dial failed"); } if(write(dfd, h->prompt, strlen(h->prompt)) < 0){ return -2; //exits("write failed"); } return dfd; } void reset_httppools(void){ hash_reset(HT_HFIEL); reset遺物箱(HFIEL箱); reset遺物箱(HMESG箱); } // char *prompt = h->format(h, i); 命一 * i int http_send(Httprq * h){ int dfd; int n = 0; h->format(h); while((dfd = dial_send(h)) < 0 ){ if(n > 3) return -1; n++; sleep(50*n); } n = 0; if (h->header){ h->header = nil; reset_httppools(); } Biobuf *bfd = Bfdopen(dfd, OREAD); int done = 0; char * line; while((!done) && (line = Brdstr(bfd, '\n', 1)) ){ if(!(h->header)){ match[0].sp = match[0].ep = 0; if(regexec(httpstatusrp, line, match, MSZ)){ /* handle http response. This must trigger first */ HTTP_mesg * response_message = get遺物(HMESG箱); char * status = copymatch_t(1); response_message->status = atoi(status); response_message->datalength = 0; response_message->chunked = 0; h->header = response_message; //free(status); } else { /* if the first line of the response is bad, then something has gone wrong. */ done = 1; } } else { /* response_message != nil */ match[0].sp = match[0].ep = 0; if(regexec(fieldrp, line, match, MSZ)){ HTTP_field * field = get遺物(HFIEL箱); field->name = copymatch_t(1); field->value = copymatch_t(2); switch(getFieldsType(field->name)){ case o_CT: // check if contains "application/json" // check if contains "charset=" and if so what? (we expect "utf-8") break; case o_D: // we can store in 命一 (json blob also has date field, so this may be redundant. break; case o_TE: h->header->datalength = -1; h->header->chunked = 1; break; case o_CL: match[0].sp = match[0].ep = 0; if(regexec(decimalrp, field->value, match, MSZ)){ h->header->datalength = atoi(field->value); } break; case o_O: // we literally don't care default: break; } hash_insert(HT_HFIEL, field->name, field); } else { match[0].sp = match[0].ep = 0; if(h->header->chunked && h->header->datalength == -1 && regexec(hexrp, line, match, MSZ)){ h->header->datalength = strtol(line,nil,16); done = 1; } } match[0].sp = match[0].ep = 0; if(h->header->datalength > 0 && regexec(blankrp, line, match, MSZ)){ done = 1; } } if(line) { free(line); //line = nil; } } // while readline if(h && h->header && h->header->datalength > 0){ line = (char*) get_temp_string(h->header->datalength + 1); n = Bread(bfd, line, h->header->datalength); if ( n != h->header->datalength ){ fprint(2, "\nHTTP read error: expected %d bytes, got %d : line - %s \n", h->header->datalength, n, line); } h->json = line; if (h->header->chunked){ //even tho we are using pools for temp strings normally we can't use it for this line see NOTE below while(line = Brdstr(bfd, '\n', 1)){ match[0].sp = match[0].ep = 0; if(regexec(hexrp, line, match, MSZ)){ int n = strtol(line,nil,16); if( n == 0 ){ Bflush(bfd); free(line); //line = nil; break; } else { //NOTE: this makes using the temp string arena across threads impossible. //give each thread that needs temp strings a thread scope instance for personal use. //only works because nothing else touches the temp from start of json blob till finish get_temp_string(n); Bread(bfd, h->json + h->header->datalength , n); h->header->datalength += n; } } free(line); //line = nil; } } h->json[h->header->datalength] = 0; } Bterm(bfd); close(dfd); return n; } int digestJSON_reply命(Httprq * rq, 命一 * n) { JSON *j_t; JSON *j; if (rq->json == nil) return -1; j = jsonparse(rq->json); if (j == nil) return -2; /* we may want to check the model in the blob to make sure it matches what we expect. (sanity check) ie 役->名 if ( ((j_t = jsonbyname(j, "model")) != nil) && (j_t->t == JSONString)) n->model = strdup(j_t->s); */ if ( ((j_t = jsonbyname(j, "created_at")) != nil) && (j_t->t == JSONString)) n->stime = strdup(j_t->s); // if ( ((j_t = jsonbyname(j, "response")) != nil) && (j_t->t == JSONString)) // n->面影 = strdup(j_t->s); //we probably don't need to do any of the following if ( ((j_t = jsonbyname(j, "done")) != nil) && (j_t->t == JSONBool)) rq->done = (j_t->n == 1)?1:0; if ( ((j_t = jsonbyname(j, "done_reason")) != nil) && (j_t->t == JSONString)) rq->done_reason = strdup(j_t->s); if ( ((j_t = jsonbyname(j, "total_duration")) != nil) && (j_t->t == JSONNumber)) rq->total_duration = floor(j_t->n); if ( ((j_t = jsonbyname(j, "load_duration")) != nil) && (j_t->t == JSONNumber)) rq->load_duration = floor(j_t->n); if ( ((j_t = jsonbyname(j, "prompt_eval_count")) != nil) && (j_t->t == JSONNumber)) rq->prompt_eval_count = floor(j_t->n); if ( ((j_t = jsonbyname(j, "prompt_eval_duration")) != nil) && (j_t->t == JSONNumber)) rq->prompt_eval_duration = floor(j_t->n); if ( ((j_t = jsonbyname(j, "eval_count")) != nil) && (j_t->t == JSONNumber)) rq->eval_count = floor(j_t->n); if ( ((j_t = jsonbyname(j, "eval_duration")) != nil) && (j_t->t == JSONNumber)) rq->eval_duration = floor(j_t->n); jsonfree(j); return 1; } int digestJSON_OAI_reply命(Httprq * rq, 命一 * n) { JSON *j_t, *j_l; JSON *j; if (rq->json == nil) return -1; j = jsonparse(rq->json); if (j == nil) return -2; if( ((j_t = jsonbyname(j, "choices")) != nil) && (j_t->t == JSONArray)){ if(!j_t->first) goto digestJSON_OAI_ERROR; j_t = j_t->first->val; if( ((j_l = jsonbyname(j_t, "message")) != nil) && (j_l->t == JSONObject)){ if( ((j_t = jsonbyname(j_l, "content")) != nil) && (j_t->t == JSONString)){ if ( insert_命一面影(n, j_t->s) == 0) goto digestJSON_OAI_ERROR; } else goto digestJSON_OAI_ERROR; if( ((j_t = jsonbyname(j_l, "reasoning_content")) != nil) && (j_t->t == JSONString)){ n->thought = find_add_string(j_t->s); } } else goto digestJSON_OAI_ERROR; } else goto digestJSON_OAI_ERROR; if(rq->think == RE_none){ if( ((j_t = jsonbyname(j, "usage")) != nil) && (j_t->t == JSONObject)){ if( ((j_t = jsonbyname(j_t, "completion_tokens")) != nil) && (j_t->t == JSONNumber)){ n->token = (int) j_t->n; } else goto digestJSON_OAI_TOKENFAILSAFE; } else goto digestJSON_OAI_TOKENFAILSAFE; } else { digestJSON_OAI_TOKENFAILSAFE: //we can't trust token count from llama.cpp if reasoning is on. n->token = strlen(tmp_get命一面影blob(n, " ", "\n", 0 ,0)) >> 2; } jsonfree(j); return 1; digestJSON_OAI_ERROR: jsonfree(j); return -1; } #define FULL役MASK 0xffffffff //return of -1 is empty json, -2 is unparsable (probably regenerate), -3 bad format (!regenerate!) int digestJSON_命三(命三 *三){ JSON *j, *j_t, *j_j; JSONEl *j_i; // char * t; int factcount = 0; if (三->tjson == nil) return -1; //the json blob is empty? why you call this? j = jsonparse(三->tjson); if (j == nil) return -2; // rerequest json object due to bad generation? /* in principle this uuid should be in the db already and should match 三->uid because this is the case we can do some kind of check on it, but for now we will simply ignore this field. if we run into issues where our data is getting mangled somehow we will look at this and maybe move to a more service based system for handling things, like using the uuid in the object hash map and then populating the returned object and having this running it a dedicated thread with the main thread doing message dispach or something. for now we aren't bothering with such parrallel implementation, and hopefully won't need to. if ( ((j_t = jsonbyname(j, "triplet_id")) != nil) && (j_t->t == JSONString)){ if(t = hash_lookup(HT_STRINGS, j_t->s)){ // 三->uid = t; } else { 三->uid = get_string(strlen(t) + 1); strcpy(三->uid, t); hash_insert(HT_STRG, 三->uid, 三->uid ); } } */ if ( ((j_t = jsonbyname(j, "summaries")) != nil) && (j_t->t == JSONArray)){ //we need to first validate the summaries entries. We must have a valid summary for every 現役 // facts are more lenient, we can validate each one and discard fully broken facts, but every persona we expect MUST // exist in the json blob, with all fields properly defined. // is potentially asking for summaries for the same triplet several times wasteful? // no. We can not allow corrupted memories into the system, we have to be strict here. uint 役mask = (FULL役MASK << 三->新現役->現役数); // what you have more than 32 present personas? what's wrong with you? j_i = j_t->first; int count = 0; while ( j_i ){ if ( ((j_j = jsonbyname(j_i->val, "役顔")) != nil) && (j_j->t == JSONString)){ char found = 0; int i = 0; 役顔 * p = (役顔 *) hash_lookup(HT_NAME_PERSONA, j_j->s); if (!p) goto digestJSON_命三regen; for(; i < 三->新現役->現役数; i++){ if (p == 三->新現役->現役[i]){ found = 1; break; } } if(!found) goto digestJSON_命三regen; 役mask = 役mask | ( 1 << i ); } else goto digestJSON_命三regen; if ( ((j_j = jsonbyname(j_i->val, "重")) != nil) && (j_j->t == JSONNumber)){ } else goto digestJSON_命三regen; if ( ((j_j = jsonbyname(j_i->val, "面影")) != nil) && (j_j->t == JSONString)){ } else goto digestJSON_命三regen; if ( ((j_j = jsonbyname(j_i->val, "声")) != nil) && (j_j->t == JSONString)){ } else goto digestJSON_命三regen; j_i = j_i->next; count++; } if(count != 三->新現役->現役数) goto digestJSON_命三regen; if(役mask != FULL役MASK) goto digestJSON_命三regen; } else goto digestJSON_命三regen; if ( ((j_t = jsonbyname(j, "facts")) != nil) && (j_t->t == JSONArray)){ for (j_i = j_t->first ; j_i ; j_i = j_i->next){ int *bail; if ( ((j_j = jsonbyname(j_i->val, "重")) != nil) && (j_j->t == JSONNumber)){ bail = &(j_j->t); //this is evil. } else continue; if ( ((j_j = jsonbyname(j_i->val, "役")) != nil) && (j_j->t == JSONString)){ //much less strict here. it should be a valid persona, but we just store the name as a string, since it doesn't need to be. } else { *bail = JSONNull; continue; } if ( ((j_j = jsonbyname(j_i->val, "type")) != nil) && (j_j->t == JSONString)){ } else { *bail = JSONNull; continue; } if ( ((j_j = jsonbyname(j_i->val, "statement")) != nil) && (j_j->t == JSONString)){ } else { *bail = JSONNull; continue; } factcount++; } if (factcount == 0 ) goto digestJSON_命三regen; } else goto digestJSON_命三regen; if ( ((j_t = jsonbyname(j, "summaries")) != nil) && (j_t->t == JSONArray)){ //passed all checks we can give it ; start commit pass j_i = j_t->first; 三->何 = get遺物A(残滓箱,三->新現役->現役数); for(int i = 0; j_i; i++){ 三->何[i].uid = get_uid(); 三->何[i].三 = 三; 三->何[i].amb = 1; j_j = jsonbyname(j_i->val, "役顔"); 三->何[i].役 = (役顔 *) hash_lookup(HT_NAME_PERSONA, j_j->s); j_j = jsonbyname(j_i->val, "重"); 三->何[i].重 = j_j->n; j_j = jsonbyname(j_i->val, "面影"); 三->何[i].面影 = get_string(strlen(j_j->s)+1); strcpy(三->何[i].面影, j_j->s); j_j = jsonbyname(j_i->val, "声"); 三->何[i].声 = get_string(strlen(j_j->s)+1); strcpy(三->何[i].声, j_j->s); j_i = j_i->next; } } if ( ((j_t = jsonbyname(j, "facts")) != nil) && (j_t->t == JSONArray)){ 欠片 *晶 = malloc(sizeof(欠片) * factcount); j_i = j_t->first; for(int i = 0 ; j_i && i < factcount; j_i = j_i->next){ if ( ((j_j = jsonbyname(j_i->val, "重")) != nil) && (j_j->t == JSONNumber)){ //if we are here it means this is a valid fact, per our evil trick 晶[i].uid = nil; 晶[i].vec = nil; 晶[i].の = nil; 晶[i].amb = 2; 晶[i].重 = j_j->n; 晶[i].三 = 三; } else continue; j_j = jsonbyname(j_i->val, "役"); 晶[i].役 = get_string(strlen(j_j->s)+1); strcpy(晶[i].役, j_j->s); j_j = jsonbyname(j_i->val, "type"); 晶[i].type = get_string(strlen(j_j->s)+1); strcpy(晶[i].type, j_j->s); j_j = jsonbyname(j_i->val, "statement"); 晶[i].statement = get_string(strlen(j_j->s)+1); strcpy(晶[i].statement, j_j->s); i++; } for(int i = 0; i < 三->新現役->現役数 ; i++){ 三->何[i].欠片count = factcount; 三->何[i].結晶 = get遺物A(欠片箱, factcount); for(int j = 0 ; j < factcount; j++){ 三->何[i].結晶[j] = 晶[j]; 三->何[i].結晶[j].uid = get_uid(); 三->何[i].結晶[j].の = &三->何[i]; } } free(晶); } jsonfree(j); return 0; digestJSON_命三regen: jsonfree(j); return -3; } char * 名L(char **names, int count) { JSON jj[3]; JSONEl j[3]; JSON t; JSONEl **tail; int i, k, duplicate, idx; t.t = JSONArray; t.first = nil; tail = &t.first; idx = 0; for (i = 0; i < count; i++) { if (!names[i]) continue; duplicate = 0; for (k = 0; k < i; k++) { if (names[k] && names[i] == names[k]) { duplicate = 1; break; } } if (duplicate) continue; jj[idx].t = JSONString; jj[idx].s = names[i]; j[idx].name = ""; j[idx].val = &jj[idx]; j[idx].next = nil; *tail = &j[idx]; tail = &j[idx].next; idx++; } return smprint("%J", &t); } char * 現役LJ(役顔 **現役, int 数) { JSON jj[32]; // bounds check or adjust if 現役 can exceed 32 JSONEl j[32]; char *formatted[32]; JSON outj; JSONEl **tail; int i, valid_count; char *out; if (数 <= 0 || !現役) return strdup("[]"); if (数 > 32) 数 = 32; outj.t = JSONArray; outj.first = nil; tail = &outj.first; valid_count = 0; for (i = 0; i < 数; i++) { if (!現役[i]) continue; if (現役[i]->血 == -1) { formatted[valid_count] = smprint("%s (%s)[%s]", 現役[i]->名, 現役[i]->性, 現役[i]->心); } else { formatted[valid_count] = smprint("%s (%s)", 現役[i]->名, 現役[i]->性); } jj[valid_count].t = JSONString; jj[valid_count].s = formatted[valid_count]; j[valid_count].name = ""; j[valid_count].val = &jj[valid_count]; j[valid_count].next = nil; *tail = &j[valid_count]; tail = &j[valid_count].next; valid_count++; } out = smprint("%J", &outj); for (i = 0; i < valid_count; i++) { if (formatted[i]) free(formatted[i]); } return out; } char * toJSON_命三(命三 * 三) { char *out, *話名, *現役, *replies_json; char *void_name; char *namelist[3] = {nil, nil, nil}; int valid_slots; int i, len; // Objects for building replies dynamically JSON reply_objs[3]; JSON reply_json_s[3]; JSON reply_json_t[3]; JSONEl reply_els[3]; JSONEl reply_pair_name[3]; JSONEl reply_pair_text[3]; JSON replies_arr; JSONEl **tail; struct { 役顔 *役; void *item; } src[3]; struct { 役顔 *役; char *text; } slots[3]; char *三fmt = "{\n" "\t\"triplet_id\": \"%s\",\n" "\t\"present_personas\": %s,\n" "\t\"present_speakers\": %s,\n" "\t\"replies\": %s\n" "}"; void_name = find_add_string("void役顔"); src[0].役 = 三->一->役; src[0].item = 三->一; src[1].役 = 三->二->役; src[1].item = 三->二; src[2].役 = 三->三->役; src[2].item = 三->三; valid_slots = 0; for (i = 0; i < 3; i++) { if (src[i].役 && src[i].役->名 != void_name) { namelist[valid_slots] = src[i].役->名; slots[valid_slots].役 = src[i].役; slots[valid_slots].text = tmp_get命一面影blob(src[i].item, nil, "\n", 1, 1); valid_slots++; } } 話名 = 名L(namelist, valid_slots); 現役 = 現役LJ(三->新現役->現役, 三->新現役->現役数); // Build 'replies' JSON array replies_arr.t = JSONArray; replies_arr.first = nil; tail = &replies_arr.first; for (i = 0; i < valid_slots; i++) { reply_json_s[i].t = JSONString; reply_json_s[i].s = slots[i].役->名; reply_json_t[i].t = JSONString; reply_json_t[i].s = slots[i].text; reply_pair_text[i].name = "text"; reply_pair_text[i].val = &reply_json_t[i]; reply_pair_text[i].next = nil; reply_pair_name[i].name = "役"; reply_pair_name[i].val = &reply_json_s[i]; reply_pair_name[i].next = &reply_pair_text[i]; reply_objs[i].t = JSONObject; reply_objs[i].first = &reply_pair_name[i]; reply_els[i].name = ""; reply_els[i].val = &reply_objs[i]; reply_els[i].next = nil; *tail = &reply_els[i]; tail = &reply_els[i].next; } replies_json = smprint("%J", &replies_arr); out = get_temp_string(0); len = snprint(out, max_temp_pool(), 三fmt, 三->uid, 現役 ? 現役 : "[]", 話名 ? 話名 : "[]", replies_json); get_temp_string(len + 1); out[len] = 0; if (話名) free(話名); if (現役) free(現役); if (replies_json) free(replies_json); return out; } int find役idxin命三(命三 *三, 役顔 *役){ if(三 && 三->新現役) for( int i = 0 ; i < 三->新現役->現役数; ++i){ if ( 役 == 三->何[i].役) return i; } return -1; } int count連中(uint bf){ int cnt; for(cnt = 0 ; bf ; ++cnt) bf &= bf - 1; return cnt; } 連中 * searchandinsert連中bybf(uint bf){ char * name = build連中key(bf); 連中 * out = hash_lookup(HT_RENSHUU, name); if ( !out ){ int count = count連中(bf); out = get遺物(連中箱); 連中 * all役 = (連中 *) hash_lookup(HT_RENSHUU, "ALL"); out->uid = get_uid(); out->現役 = get遺物A(役顔p箱, count); out->現役数 = count; for(int i = 0, j = 0; i < all役->現役数 && j < count ; ++i){ if (bf & (1<現役[j++] = all役->現役[i]; } } hash_insert(HT_RENSHUU, name, out); } return out; } char * build連中key(uint bf){ 連中 * all = (連中 *) hash_lookup(HT_RENSHUU, "ALL"); char * start; char * trail = start = get_temp_string(0); char * bufend = trail + max_temp_pool(); for(int i = 0; all && (i < all->現役数) && bf ; ++i){ if( bf & 1) trail = seprint(trail, bufend, "%s", all->現役[i]->名); bf = bf >> 1; } get_temp_string(trail - start + 1); start = find_add_string(start); return start; } //this builds the current cannon string for a triplet regardless of the list order char * generate連中key(連中 * ren, int aslist){ 連中 * all = (連中 *) hash_lookup(HT_RENSHUU, "ALL"); char * start; char * trail = start = get_temp_string(0); char * bufend = trail + max_temp_pool(); int count = 0; for(int i = 0; all && (i < all->現役数); ++i){ for(int j = 0; ren && (j < ren->現役数) ; ++j){ if (all->現役[i]->名 == ren->現役[j]->名 ){ //point cmp works since these are cannonized strings trail = seprint(trail, bufend, "%s%s", aslist?" ":"", all->現役[i]->名); ++count; } } } if(count == ren->現役数){ //all the personae are in the "all" list //we only need to protect this temp string on this path. get_temp_string(trail - start + 1); start = find_add_string(start); return start; } else { //contains void actors. we don't want this to be able to be looked up except if something knows this is what it needs return ren->uid; } } char * toslov命一(命一 * n){ char *out; if(n == nil) return find_add_string(""); if(n->uid == find_add_string("void命一")) return find_add_string(""); if (n->面影数 < 1) return find_add_string(""); char * 面影 = tmp_get命一面影blob(n," ","\n", 0, 0); if(!面影 || (strlen(面影) <= 2 )) return find_add_string(""); out = get_temp_string(0); int i = snprint(out, max_temp_pool(), "\n命一 %s\n before : %s\n 役 : %s\n 面影\n%s token : %d\n time : %lld\n after : %s\n atbot : %s\n atmid : %s\n attop : %s\n" , n->uid?n->uid:"nil", (n->before&&n->before->uid)?n->before->uid:"void命一", (n->役&&n->役->名)?n->役->名:"nil", 面影, n->token, n->time, (n->after&&n->after->uid)?n->after->uid:"void命一", n->atbot && n->atbot->uid?n->atbot->uid:"void命三", n->atmid && n->atmid->uid?n->atmid->uid:"void命三", n->attop && n->attop->uid?n->attop->uid:"void命三"); get_temp_string(i+1); out[i] = 0; return out; } char * toslov連中(連中 * n){ char * out; char *t現役list = nil; if(n == nil) return find_add_string(""); if(n->uid == find_add_string("void連中")) return find_add_string(""); if(n->uid == find_add_string("ALL")) //synthetic all return find_add_string(""); if (n->現役 && n->現役数>0){ t現役list = get_temp_string(30*(n->現役数 +5)); t現役list[0] = 0 ; char *writehead = t現役list; for(int i = 0 ; i < n->現役数 ; ++i){ if(n->現役[i] && n->現役[i]->名) writehead = seprint(writehead, t現役list+(30*(n->現役数 +5)), " %s\n", n->現役[i]->名); } } out = get_temp_string(0); int i = snprint(out, max_temp_pool(), "連中 %s\n 現役\n%s\n", n->uid, t現役list); get_temp_string(i+1); out[i] = 0; return out; } char * toslov役顔 (役顔 *n){ char * out; if(n == nil) return find_add_string(""); if(n->名 == find_add_string("void役顔")) return find_add_string(""); //心 - read in from name.md //魂 - completely synthetic char * t = tmp_getstrarrayblob(n->名s, n->名sn, " ", "\n", 1, 0); out = get_temp_string(0); int i = snprint(out, max_temp_pool(), "\n役顔 %s\n 血 : %d\n 性 : %s\n 地 : %s\n 名s\n%s\n", n->名, n->血, n->性, n->地, t); get_temp_string(i+1); out[i] = 0; return out; } char * toslov命三(命三 * n){ char *out; char *何list; if(n == nil) return find_add_string(""); if(n->uid == find_add_string("void命三")) return find_add_string(""); //何 if (n->何 && n->新現役->現役数>0){ 何list = get_temp_string(25 * (n->新現役->現役数 +2)); 何list[0] = 0 ; // incase count is more than 1 and we can't read any uid. this will be an empty string char* writehead = 何list; for(int i = 0 ; i < n->新現役->現役数 ; ++i){ if(n->何[i].uid) writehead = seprint(writehead, 何list+(25 * (n->新現役->現役数 +2)), " %s\n", n->何[i].uid); } } else { return find_add_string(""); } 連中 * cannon中 = hash_lookup(HT_RENSHUU, generate連中key(n->新現役, 0)); out = get_temp_string(0); int i = snprint(out, max_temp_pool(), "\n命三 %s\n 一 : %s\n 二 : %s\n 三 : %s\n 現役 : %s\n 何\n%s\n" , n->uid?n->uid:"void命三", (n->一 && n->一->uid)?n->一->uid:"void命一", (n->二 && n->二->uid)?n->二->uid:"void命一", (n->三 && n->三->uid)?n->三->uid:"void命一", (cannon中 && cannon中->uid)?cannon中->uid:"void連中", 何list); get_temp_string(i+1); out[i] = 0; return out; } char * vectortostring(float *vec, uint d){ char* b = get_temp_string(d*9+1); u32int *v = (u32int*)vec; for(int i=0; iuid == find_add_string("voidvwrap")) return find_add_string(""); if(n->d == 0) return find_add_string(""); if(n->v == nil) return find_add_string(""); char *out = get_temp_string(0); char *writehead; char *end = out + max_temp_pool(); writehead = seprint(out, end, "vwrap %s\n d : %d\n v : ", n->uid, n->d); writehead = vectortostringbuf(writehead, end, n->v, n->d); writehead = seprint(writehead, end, "\n"); get_temp_string(writehead-out + 1); return out; } char * toslov残滓(残滓 * n){ char *out; char *結晶list = nil; char *vecstring = nil; if(n == nil) return find_add_string(""); if(n->uid == find_add_string("void残滓")) return find_add_string(""); if (n->vec && n->vec->uid){ vecstring = n->vec->uid; } if (n->結晶 && n->欠片count>0){ 結晶list = get_temp_string(25 * (n->欠片count +1)); 結晶list[0] = 0 ; // incase count is more than 1 and we can't read any uid. this will be an empty string char* writehead = 結晶list; for(int i = 0 ; i < n->欠片count ; ++i){ if(n->結晶 && n->結晶[i].uid){ writehead = seprint(writehead, 結晶list+(25 * (n->欠片count +1)), " %s\n", n->結晶[i].uid); } } } out = get_temp_string(0); int i = snprint(out, max_temp_pool(), "\n残滓 %s\n 結晶\n%s\n 欠片count : %d\n 三 : %s\n 声 : %s\n 面影 : %s\n 役 : %s\n 重 : %f\n vector : %s\n" , n->uid?n->uid:"void残滓", 結晶list?結晶list:"", n->欠片count?n->欠片count:0, n->三 && n->三->uid?n->三->uid:"void命三", n->声?n->声:"", n->面影?n->面影:"", n->役?n->役->名:"void役顔", n->重, vecstring?vecstring:"voidvwrap" ); get_temp_string(i+1); out[i] = 0; return out; } char * toslov欠片(欠片 * n){ char *out; char *vecstring = nil; if(n == nil) return find_add_string(""); if(n->uid == find_add_string("void欠片")) return find_add_string(""); if (n->vec && n->vec->uid){ vecstring = n->vec->uid; } out = get_temp_string(0); int i = snprint(out, max_temp_pool(), "\n欠片 %s\n 役 : %s\n type : %s\n statement : %s\n 重 : %f\n 三 : %s\n の : %s\n vector : %s\n" , n->uid?n->uid:"void欠片", n->役?n->役:"void役顔", n->type?n->type:"nil", n->statement?n->statement:"nil", n->重, n->三 && n->三->uid?n->三->uid:"void命三", n->の && n->の->uid?n->の->uid:"void残滓", vecstring?vecstring:"voidvwrap" ); get_temp_string(i+1); out[i] = 0; return out; } ///Loading starts here to the end T-T #define SLTOFST 4 int hydratevector(float * vector, int d, char * tvecstr){ if(!(tvecstr && (strlen(tvecstr) >= ((9 * d) - 1)) )){ return -1; } tvecstr = strtok(tvecstr, ","); int j; for(j = 0 ; (j < d) && tvecstr; j++){ u32int vali = strtoul(tvecstr, nil, 16); u32int *valip = &vali; float * valp = (float *)valip; vector[j] = *valp; tvecstr = strtok(nil, ","); } return j; } int hydrate役顔(HashTable *curelement, HashTable * precommitdict, char * line, char *types[]){ char * listitemkeybuffer = get_temp_string(15); char *listitemfmt = "list_%0l5d"; char *syspmtfmt = "./%s.md"; int i = TYAKU; char * t名 = find_add_string(hash_lookup(curelement, types[i]+SLTOFST)); if(! t名 ){ fprint(2, "役顔 name not found, before line : %s\n", line); return -1; //this is a true "not found" so we do return -1 (we have no name to link to void object) } 役顔 * 役 = get遺物(役顔箱); 役->名 = t名; char * chi = (char *)hash_lookup(curelement, "血"); if(chi){ 役->血 = atoi(chi); } else { 役->血 = 100; } 役->性 = find_add_string(hash_lookup(curelement, "性")); 役->地 = find_add_string(hash_lookup(curelement, "地")); 役->名sn = *((int*)hash_lookup(curelement, "list")); char * syspmtpath = get_temp_string(0); int size = sprint(syspmtpath, syspmtfmt, 役->名, types[i]+SLTOFST); get_temp_string(size+1); Biobuf * sysprompt = Bopen(syspmtpath, OREAD); if(sysprompt){ char * tk = get_temp_string(0); int len = Bread(sysprompt, tk, max_temp_pool()); get_temp_string(len+1); tk[len] = 0; Bterm(sysprompt); 役->心 = find_add_string(tk); } else { 役->心 = find_add_string(""); }//end sysprompt 役->名s = get遺物A(役顔p箱, 役->名sn); for ( int j = 0; j < 役->名sn ; j++){ snprint(listitemkeybuffer, 15, listitemfmt, j); char * tname = (char *) hash_lookup(curelement, listitemkeybuffer); if ( !tname ){ 役->名s[j] = nil; continue; } 役->名s[j] = find_add_string(tname); //cannonized in HT_NAME_PERSONA //the aliases are needed because the summarizer can't be trusted to be consistant. } hash_insert(precommitdict, 役->名, 役); return 1; } int hydrate連中(HashTable *curelement, HashTable * precommitdict, char * line, char *types[]){ char * listitemkeybuffer = get_temp_string(15); char *listitemfmt = "list_%0l5d"; int i = TRENSHU; char * t中uid = find_add_string(hash_lookup(curelement, types[i]+SLTOFST)); if(! t中uid ){ fprint(2, "連中 uid not found, before line : %s\n", line); return -1; //no name to commit, actual failure } int 中n; 中n = *((int *)hash_lookup(curelement, "list")); //we have to assume that the number we read here is the actual count if (中n == 0) { //woops void //it can't be the real void, just an empty one //but since it is empty it will be merged with the void hash_insert(precommitdict, t中uid, hash_lookup(precommitdict, types[i])); return 0; } 連中 * 中 = get遺物(連中箱); 中->uid = t中uid; 中->現役 = get遺物A(役顔p箱, 中n); 中->現役数 = 中n; 役顔 * voiditem = hash_lookup(precommitdict, types[TYAKU]); for ( int j = 0; j < 中n ; j++){ 中->現役[j] = voiditem; snprint(listitemkeybuffer, 15, listitemfmt, j); char * tname = (char *) hash_lookup(curelement, listitemkeybuffer); if ( !tname ){ fprint(2, "listitem read failure 連中 %s\n", 中->uid); continue; } 役顔 * curjtem = (役顔 *)hash_lookup(precommitdict, tname); if ( curjtem ) 中->現役[j] = curjtem; else fprint(2, "listitem read failure 連中 %s - no persona %s\n", 中->uid, tname); /* the philosophy for this version was /one invalid member kills the cohort This feels high handed. but we may need to revert to this if there are problems with leaving voids in the cohort snprint(listitemkeybuffer, 15, listitemfmt, j); char * tname = (char *) hash_lookup(curelement, listitemkeybuffer); if ( !tname ){ fprint(2, "listitem read failure 連中 %s\n", 中->uid); hash_insert(precommitdict, t中uid, hash_lookup(precommitdict, types[i])); return 0; } 中->現役[j] = (役顔 *)hash_lookup(precommitdict, tname); if (!中->現役[j]){ fprint(2, "listitem read failure 連中 %s - no persona %s\n", 中->uid, tname); hash_insert(precommitdict, t中uid, hash_lookup(precommitdict, types[i])); return 0; } */ } hash_insert (precommitdict, t中uid, 中); return 1; } int hydrate命一(HashTable *curelement, HashTable * precommitdict, char * line, char *types[]){ char * listitemkeybuffer = get_temp_string(15); char *listitemfmt = "list_%0l5d"; int i = TINOICHI; char * t一uid = find_add_string(hash_lookup(curelement, types[i]+SLTOFST)); if(! t一uid ){ fprint(2, "命一 uid not found, before line : %s\n", line); return -1; } 命一 *一 = get遺物(命一箱); 一->uid = t一uid; 一->token = atoi((char *)hash_lookup(curelement, "token")); 一->time = atoll((char *)hash_lookup(curelement, "time")); 一->役 = (役顔 *) hash_lookup(precommitdict, hash_lookup(curelement, "役")); if(!一->役){ 一->役 = (役顔 *) hash_lookup(precommitdict, types[TYAKU]); } 一->面影数 = *((int *)hash_lookup(curelement, "list")); 一->面影 = get遺物A(語録箱, 一->面影数); for ( int j = 0; j < 一->面影数 ; j++){ snprint(listitemkeybuffer, 15, listitemfmt, j); char * tstr = (char *) hash_lookup(curelement, listitemkeybuffer); //fprint(2, "inserting line into %s's 命 - %s\n", 一->役->名, tstr); if ( !tstr ){ //may actually happen normally 一->面影[j] = find_add_string(""); } else { 一->面影[j] = find_add_string(tstr); } } 一->thought = nil; hash_insert(precommitdict, 一->uid, 一); return 1; } int hydrate命三(HashTable *curelement, HashTable * precommitdict, char * line, char *types[]){ char * listitemkeybuffer = get_temp_string(15); char *listitemfmt = "list_%0l5d"; int i = TINOSAN; char * t三uid = find_add_string(hash_lookup(curelement, types[i]+SLTOFST)); if(! t三uid ){ fprint(2, "命三 uid not found, before line : %s\n", line); return -1; } //fprint(2, "hydrate %s\n", t三uid); 命三 * 三 = get遺物(命三箱); 三->uid = t三uid; 三->一 = (命一 *)hash_lookup( precommitdict, hash_lookup(curelement, "一")); 三->二 = (命一 *)hash_lookup( precommitdict, hash_lookup(curelement, "二")); 三->三 = (命一 *)hash_lookup( precommitdict, hash_lookup(curelement, "三")); //these /can/ just be void objects gracefully in normal operation so no reporting that if(!三->一) 三->一 = (命一 *) hash_lookup(precommitdict, types[TINOICHI]); if(!三->二) 三->二 = (命一 *) hash_lookup(precommitdict, types[TINOICHI]); if(!三->三) 三->三 = (命一 *) hash_lookup(precommitdict, types[TINOICHI]); 三->新現役 = (連中*)hash_lookup( precommitdict, hash_lookup(curelement, "現役") ); if(!三->新現役){ fprint(2, "現役 not found in 命三 %s using void object\n", 三->uid); 三->新現役 = (連中*) hash_lookup(precommitdict, types[TYAKU]); } int 三n; 三n = *((int *)hash_lookup(curelement, "list")); if ( 三->新現役->現役数 != 三n ) { fprint(2, "Number of 何 in 命三 %s doesn't match count in 現役 commiting as void\n", 三->uid); hash_insert(curelement, t三uid, hash_lookup(precommitdict, types[i])); return 0; } 三->何 = get遺物A(残滓箱, 三n); 残滓 *voiditem = hash_lookup(precommitdict, types[TGINSHI]); for ( int j = 0; j < 三n ; j++){ 三->何[j] = *voiditem; // incase we can't read uid, or there is no link back into a read one. snprint(listitemkeybuffer, 15, listitemfmt, j); char * tguid = (char *) hash_lookup(curelement, listitemkeybuffer); //fprint(2, "checking for %s - got %s\n", listitemkeybuffer, tguid); if ( !tguid ){ fprint(2, "listitem read failure 命三 %s : setting memory to void object data\n", 三->uid); } else { tguid = find_add_string(tguid); //fprint(2, "added to precommit\n"); hash_insert(precommitdict, tguid, &(三->何[j])); } } hash_insert(precommitdict, 三->uid, 三); return 1; } int hydratevwrap(HashTable *curelement, HashTable * precommitdict, char * line, char *types[]){ int i = TVWRAP; char * tvwrapuid = find_add_string(hash_lookup(curelement, types[i]+SLTOFST )); if(! tvwrapuid ){ fprint(2, "vwrap uid not found, before line : %s\n", line); return -1; } vwrapper * vwrap = get遺物(vwrap箱); vwrap->uid = tvwrapuid; vwrap->d = atoi((char *)hash_lookup(curelement, "d")); if (vwrap->d > 0){ vwrap->v = get遺物A(VEC箱, vwrap->d); char * tvecstr = hash_lookup(curelement, "v"); int vret = hydratevector(vwrap->v, vwrap->d, tvecstr); if(vret != vwrap->d){ fprint(2, "vector read failure vwrap %s, setting as void\n", vwrap->uid); vwrap = hash_lookup(precommitdict, types[TVWRAP]); } } else { fprint(2, "vector dim = 0 in %s, setting as void\n", vwrap->uid); vwrap = hash_lookup(precommitdict, types[TVWRAP]); } hash_insert (precommitdict, tvwrapuid, vwrap); return 1; } int hydrate残滓(HashTable *curelement, HashTable * precommitdict, char * line, char *types[]){ char * listitemkeybuffer = get_temp_string(15); char *listitemfmt = "list_%0l5d"; int i = TGINSHI; char * t残滓uid = find_add_string(hash_lookup(curelement, types[i]+SLTOFST )); if(! t残滓uid ){ fprint(2, "残滓 uid not found, before line : %s\n", line); return -1; } 残滓 * 残 = (残滓 *)hash_lookup(precommitdict, t残滓uid); //fprint(2, "looking up %s\n", t残滓uid); if (!残){ fprint(2, "残滓 %s has no parent: commiting as null object\n", t残滓uid); //*残 = *((残滓 *)hash_lookup(precommitdict, types[i])); hash_insert(precommitdict, t残滓uid, hash_lookup(precommitdict, types[i])); return 0; } 残->役 = (役顔 *) hash_lookup(precommitdict, (char *)hash_lookup(curelement, "役")); if ((! 残->役) ||(残->役 == hash_lookup(precommitdict, types[TYAKU]))){ fprint(2, "役 in 残滓 %s bad or missing using void\n", 残->uid); *残 = *((残滓 *)hash_lookup(precommitdict, types[i])); hash_insert(precommitdict, t残滓uid, hash_lookup(precommitdict, types[i])); return 0; } 残->amb = 1; 残->uid = t残滓uid; 残->欠片count = atoi((char *)hash_lookup(curelement, "欠片count")); 残->声 = find_add_string((char *)hash_lookup(curelement, "声")); 残->面影 = find_add_string((char *)hash_lookup(curelement, "面影")); 残->重 = atof((char *)hash_lookup(curelement, "重")); char * tvecstr = hash_lookup(curelement, "vector"); 残->vec = (vwrapper *) hash_lookup(precommitdict, tvecstr); if (! 残->vec){ // char * dstr = (char *)hash_lookup(curelement, "d"); int d= 0; if (dstr) d = atoi(dstr); if(! (strlen(tvecstr) >= ((9 * d) - 1))){ fprint(2, "vector/vwrapper in 残滓 %s bad or missing using void\n", 残->uid); 残->vec = hash_lookup(precommitdict, types[TVWRAP]); } else { 残->vec = get遺物(vwrap箱); 残->vec->v = get遺物A(VEC箱, d); 残->vec->d = d; 残->vec->uid = get_uid(); char * pvecstr = find_add_string(tvecstr); int vret = hydratevector(残->vec->v, 残->vec->d, tvecstr); if (vret != d){ fprint(2, "vector read failure 残滓 %s, setting as void\n", 残->uid); 残->vec = hash_lookup(precommitdict, types[TVWRAP]); } else { hash_insert(precommitdict, pvecstr, 残->vec); hash_insert(precommitdict, 残->vec->uid, 残->vec); } } } 残->三 = (命三 *) hash_lookup(precommitdict, (char *)hash_lookup(curelement, "三")); if (! 残->三){ fprint(2, "三 in 残滓 %s bad or missing using void\n", 残->uid); 残->三 = hash_lookup(precommitdict, types[TINOSAN]); } int 残n = *((int *)hash_lookup(curelement, "list")); if ( 残->欠片count != 残n ) { fprint(2, "Number of 結晶 in 残滓 %s doesn't match 欠片count, taking the smaller\n", 残->uid); 残n = 残->欠片count = (残->欠片count < 残n) ? 残->欠片count : 残n; } 残->結晶 = get遺物A(欠片箱, 残n); 欠片 *voiditem = (欠片*) hash_lookup(precommitdict, types[TKAKERA]); for ( int j = 0; j < 残n ; j++){ 残->結晶[j] = *voiditem; snprint(listitemkeybuffer, 15, listitemfmt, j); char * tguid = (char *) hash_lookup(curelement, listitemkeybuffer); if ( !tguid ){ fprint(2, "listitem read failure 残滓 %s : setting memory to void object data\n", 残->uid); } else { tguid = find_add_string(tguid); hash_insert (precommitdict, tguid, &(残->結晶[j])); } } hash_insert (precommitdict, t残滓uid, 残); return 1; } int hydrate欠片(HashTable *curelement, HashTable * precommitdict, char * line, char *types[]){ int i = TKAKERA; char * t欠片uid = find_add_string(hash_lookup(curelement, types[i]+SLTOFST)); if(! t欠片uid ){ fprint(2, "欠片 uid not found, before line : %s\n", line); return -1; } 欠片 * 片 = (欠片 *)hash_lookup(precommitdict, t欠片uid); if (!片){ fprint(2, "欠片 %s has no parent: commiting as null object\n", t欠片uid); //*片 = *((欠片 *)hash_lookup(precommitdict, types[i])); hash_insert(precommitdict, t欠片uid, hash_lookup(precommitdict, types[i])); return 0; } 片->amb = 2; 片->uid = t欠片uid; 片->重 = atof((char *)hash_lookup(curelement, "重")); char * tvecstr = hash_lookup(curelement, "vector"); 片->vec = (vwrapper *) hash_lookup(precommitdict, tvecstr); if (! 片->vec){ // char * dstr = (char *)hash_lookup(curelement, "d"); int d = 0; if (dstr) d = atoi(dstr); if(! (strlen(tvecstr) >= ((9 * d) - 1))){ fprint(2, "vector/vwrapper in 欠片 %s bad or missing using void\n", 片->uid); 片->vec = hash_lookup(precommitdict, types[TVWRAP]); } else { //fprint(2, "Fallback path : d(%d) v(%s)\n\n", d, tvecstr); 片->vec = get遺物(vwrap箱); 片->vec->v = get遺物A(VEC箱, d); 片->vec->d = d; 片->vec->uid = get_uid(); char * pvecstr = find_add_string(tvecstr); int vret = hydratevector(片->vec->v, d, tvecstr); if (vret != d){ fprint(2, "vector read failure 欠片 %s, setting as void\n", 片->uid); 片->vec = hash_lookup(precommitdict, types[TVWRAP]); } else { hash_insert(precommitdict, pvecstr, 片->vec); hash_insert(precommitdict, 片->vec->uid, 片->vec); } } } 片->三 = (命三 *) hash_lookup(precommitdict, (char *)hash_lookup(curelement, "三")); if (! 片->三){ fprint(2, "三 in 欠片 %s bad or missing : giving null object\n", 片->uid); 片->三 = (命三 *) hash_lookup(precommitdict, types[TINOSAN]); } 片->の = (残滓 *) hash_lookup(precommitdict, (char *)hash_lookup(curelement, "の")); if (! 片->の){ fprint(2, "残滓(の) in 残滓 %s bad or missing : giving null object\n", 片->uid); 片->の = (残滓 *)hash_lookup(precommitdict, types[TGINSHI]); } 片->役 = find_add_string((char *)hash_lookup(curelement, "役")); //just a string 片->type = find_add_string((char *)hash_lookup(curelement, "type")); 片->statement = find_add_string((char *)hash_lookup(curelement, "statement")); hash_insert (precommitdict, 片->uid, 片); return 1; } typedef struct slovloadctx{ char * id; Biobuf ** bios; HashTable * precommitdict; HashTable * curelement; int i; char *** items; int * itemscnt; char ** types; char ** listids; char * line; int (*lhf)(struct slovloadctx *); }slovloadctx; //precommitdict will hold all types that need to be referenced for building, but for all these we can't be commiting them //to the global hashtables yet int loaderhydrate(slovloadctx * ctx){ HashTable *curelement = ctx->curelement; HashTable * precommitdict = ctx->precommitdict; int i = ctx->i; char * line = ctx->line; char ** types = ctx->types; if(!line) line = ""; if(! strcmp( hash_lookup(curelement, types[i]+SLTOFST), types[i])) return 0; //this is a savefile copy of the void object switch(i) { case TYAKU: return hydrate役顔(curelement, precommitdict, line, types); case TRENSHU: return hydrate連中(curelement, precommitdict, line, types); case TINOICHI: return hydrate命一(curelement, precommitdict, line, types); case TINOSAN: return hydrate命三(curelement, precommitdict, line, types); case TVWRAP: return hydratevwrap(curelement, precommitdict, line, types); case TGINSHI: return hydrate残滓(curelement, precommitdict, line, types); case TKAKERA: return hydrate欠片(curelement, precommitdict, line, types); default: return -1; } } int loaderstitch(slovloadctx * ctx){ HashTable *curelement = ctx->curelement; HashTable * precommitdict = ctx->precommitdict; int i = ctx->i; if ( i != TINOICHI ) return -1; //we only care about the 命一 case char ** types = ctx->types; char * key = hash_lookup(curelement, types[i]+SLTOFST); if(! strcmp( key, types[i])) return 0; //this is the savefile copy of the void object 命一 *void一 = (命一 *)hash_lookup(precommitdict, types[TINOICHI]); 命三 *void三 = (命三 *)hash_lookup(precommitdict, types[TINOSAN]); if (!void一 || !void三) return -1; 命一 *命 = hash_lookup(precommitdict, key); if (!命 || void一 == 命) return 0; 命->before = (命一 *) hash_lookup(precommitdict, hash_lookup(curelement, "before")); 命->after = (命一 *) hash_lookup(precommitdict, hash_lookup(curelement, "after")); 命->atbot = (命三 *) hash_lookup(precommitdict, hash_lookup(curelement, "atbot")); 命->atmid = (命三 *) hash_lookup(precommitdict, hash_lookup(curelement, "atmid")); 命->attop = (命三 *) hash_lookup(precommitdict, hash_lookup(curelement, "attop")); if ( ! 命->before ) 命->before = void一; if ( ! 命->after ) 命->after = void一; if ( ! 命->atbot ) 命->atbot = void三; if ( ! 命->atmid ) 命->atmid = void三; if ( ! 命->attop ) 命->attop = void三; return 1; } 役顔 * createvoid役顔(void){ 役顔 * out = get遺物(役顔箱); out->血 = -99; out->名 = find_add_string("void役顔"); out->名s = nil; //can be left nil with 名sn == 0 functions that read these always for < 名sn out->名sn = 0; out->性 = find_add_string("void"); out->心 = find_add_string("void"); out->地 = find_add_string("void"); out->魂 = nil; return out; } 連中 * createvoid連中(void){ 連中 * out = get遺物(連中箱); out->uid = find_add_string("void連中"); out->現役 = nil; //can be left nil with 現役数 == 0 functions that read these always for < 現役数 out->現役数 = 0; return out; } 命一 * createvoid命一(void){ 命一 * out = get遺物(命一箱); out->uid = find_add_string("void命一"); out->before = nil; out->役 = nil; out->面影 = nil; out->面影数 = 0; out->token = 0; out->time = 0; out->stime = find_add_string("void"); out->after = nil; out->atbot = nil; out->atmid = nil; out->attop = nil; out->thought = nil; return out; } 命三 * createvoid命三(void){ 命三 * out = get遺物(命三箱); out->uid = find_add_string("void命三"); out->一 = nil; out->二 = nil; out->三 = nil; out->新現役 = nil; out->何 = nil; out->tjson = find_add_string("void"); return out; } vwrapper * createvoidvwrapper(void){ vwrapper * out = get遺物(vwrap箱); out->uid = find_add_string("voidvwrap"); out->d = 0; out->v = nil; return out; } 残滓 * createvoid残滓(void){ 残滓 * out = get遺物(残滓箱); out->amb = 0; out->vec = nil; out->uid = find_add_string("void残滓"); out->重 = 0.0f; out->三 = nil; out->役 = nil; out->面影 = find_add_string("void"); out->声 = find_add_string("void"); out->結晶 = nil; out->欠片count = 0; return out; } //need to stitch the 三 and 役 to their void objects 欠片 * createvoid欠片(void){ 欠片 * out = get遺物(欠片箱); out->amb = 0; out->vec = nil; out->uid = find_add_string("void欠片"); out->重 = 0.0f; out->三 = nil; out->役 = find_add_string("void"); out->type = find_add_string("void"); out->statement = find_add_string("void"); out->の = nil; return out; } void createvoidobjects(HashTable * objecttable){ 役顔 * 役 = createvoid役顔(); 連中 * 中 = createvoid連中(); 命一 * 一 = createvoid命一(); 命三 * 三 = createvoid命三(); vwrapper* v = createvoidvwrapper(); 残滓 * 滓 = createvoid残滓(); 欠片 * 片 = createvoid欠片(); //no fix needed for 役顔 連中 or vwrapper //fix 命一 一->before = 一->after = 一; 一->atbot = 一->atmid = 一->attop = 三; 一->役 = 役; //fix 命三 三->一 = 三->二 = 三->三 = 一; 三->新現役 = 中; 三->何 = 滓; //fix 残滓 滓->結晶 = 片; 滓->役 = 役; 滓->三 = 三; 滓->vec = v; //fix 欠片 片->の = 滓; 片->三 = 三; 片->vec = v; hash_insert(objecttable, 役->名, 役); hash_insert(objecttable, 中->uid, 中); hash_insert(objecttable, 一->uid, 一); hash_insert(objecttable, 三->uid, 三); hash_insert(objecttable, v->uid, v); hash_insert(objecttable, 滓->uid, 滓); hash_insert(objecttable, 片->uid, 片); } int slovloadstatemachine(slovloadctx * ctx) { char *listitemfmt = "list_%0l5d"; typedef enum loadstate { l_TOP = 1, l_INNODE = 1<<1, l_INLIST = 1<<2, l_err = 0 } loadstate; loadstate readstate = l_TOP; Biobuf ** bios = ctx->bios; int i = ctx->i; char * id = ctx->id; char ** types = ctx->types; HashTable * curelement = ctx->curelement; char ** items = ctx->items[i]; int itemscnt = ctx->itemscnt[i]; char * listid = ctx->listids[i]; int (*lhf)(slovloadctx * ) = ctx->lhf; int listcount = 0; // a pointer to this is used in the hashmap for current element if(! bios[i]){ //missing 役顔 is hard failure - the rest can be empty if(i == TYAKU){ fprint(2, "./Memories/%s/役顔.slov missing. Must have persona definitions\n", id); exits("no 役顔.slov in current head. unable to load memories"); } //vwrap.slov may be missing. will migrate old vector format. if (i == TVWRAP){ fprint(2, "New vector format not found. Will attempt fallback to migrate\n"); return 0; } //we read in dependancy order so if 1 is missing information in 2 becomes meaningless and so on. return -1; } char * line; line = Brdstr(bios[i], '\n', 1); if(!line){ if(i == TYAKU){ fprint(2, "./Memories/%s/役顔.slov unable to be read.\n", id); exits("Unreadable 役顔.slov in current head. unable to load memories"); } if (i == TVWRAP){ fprint(2, "New vector format not found. Will attempt fallback to migrate\n"); return 0; } return -1; } do{ // fprint(2, "processing line (%s) \n", line); match[0].sp = match[0].ep = 0; if(regexec(sloveofrp, line, match, MSZ)){ free(line); continue; //comment line } match[0].sp = match[0].ep = 0; if(regexec(slovnamerp, line, match, MSZ)){ //we are always interested in new top level if(readstate & (l_INNODE | l_INLIST) ){ //validate all data members we need to if (! hash_lookup(curelement, types[i]+SLTOFST) ) goto LOADELEMENTFAILEDPARSE; for(int j = 0 ; j < itemscnt; ++j){ if(i>=TGINSHI && j+1==itemscnt) //horrible patch to make 'd' optional on GINSHI and KAKERA continue; if( !hash_lookup(curelement, items[j])){ fprint(2, "item [%s] not found\n", items[j]); goto LOADELEMENTFAILEDPARSE;} } int * ct = (int *)hash_lookup(curelement, "list"); if(listid && (!ct || *ct <= 0)){ fprint(2, "listcount too low\n"); goto LOADELEMENTFAILEDPARSE;} ctx->line = line; int hydret = lhf(ctx); if ( 0 == hydret ){ fprint(2, "Node constructed as void. Saving will not include node: above line %s \n", line); } if ( -1 == hydret ){ LOADELEMENTFAILEDPARSE: ; fprint(2, "Failed to parse entry: above line %s \n", line); } hash_reset (curelement); reset_temp_pool(); } readstate = l_INNODE; char * readtype = copymatch_t(1); // _t temp pool stored if(strcmp(readtype, types[i]+SLTOFST)){ fprint(2, " %s.slov contains %s node. Skipping\n", types[i]+SLTOFST, readtype); readstate = l_TOP; free(line); continue; }//wait till after this check to commit the guid to our permstore since we are skipping char * readval = copymatch(2); //copymatch perm stores and deduplicates strings! hash_insert(curelement, readtype, readval); // fprint(2, " new item %s \n", readval); free(line); continue; } match[0].sp = match[0].ep = 0; if( (readstate&(l_INNODE|l_INLIST)) && regexec(slovkvrp, line, match, MSZ)){ readstate = l_INNODE; char * readtype = copymatch_t(1); // _t temp pool stored for (int j = 0; j < itemscnt ; ++j){ if (!strcmp(readtype, items[j]) ){ char * readval = copymatch_t(2); hash_insert(curelement, readtype, readval); // fprint(2, " key %s - val %s \n", readtype, readval); break; } } free(line); continue; } match[0].sp = match[0].ep = 0; if( (readstate&(l_INNODE)) && regexec(slovlkrp, line, match, MSZ)){ char * readtype = copymatch_t(1); // _t temp pool stored if (listid && (strcmp(readtype, listid)==0)){ readstate = l_INLIST; listcount = 0; hash_insert(curelement, "list", &listcount); } else { fprint(2, "Malformed memory file %s.slov GUID: %s - list invalid [%s] - looking for [%s]\n", types[i]+SLTOFST, hash_lookup(curelement, types[i]+SLTOFST), readtype, listid); } // fprint(2, " newlist %s \n", readtype); free(line); continue; } match[0].sp = match[0].ep = 0; if( (readstate&(l_INLIST)) && regexec(slovlvrp, line, match, MSZ)){ //we are always interested in new top level int * ct = (int *) hash_lookup(curelement, "list"); if (! ct){ fprint(2, "Malformed memory file %s.slov GUID: [%s] - list item\n", types[i]+SLTOFST, hash_lookup(curelement, types[i]+SLTOFST)); continue; } char *listitemkeybuffer = get_temp_string(15); char * readval = copymatch_t(1); // _t temp pool stored snprint(listitemkeybuffer, 15, listitemfmt, *ct); hash_insert(curelement, listitemkeybuffer, readval); // fprint(2, "listcount(%d - ",*ct); (*ct)++; // fprint(2, "%d) listitem - %s \n",*ct, readval); } //blank or malformed line free(line); continue; } while( line = Brdstr(bios[i], '\n', 1) ); {// we just EOFed so we need to add the last item //validate all data members we need to if (! hash_lookup(curelement, types[i]+SLTOFST) ) goto LOADELEMENTFAILEDPARSE2; for(int j = 0 ; j < itemscnt; ++j){ if(i>=TGINSHI && j+1==itemscnt) //horrible patch to make 'd' optional on GINSHI and KAKERA continue; if( !hash_lookup(curelement, items[j])) goto LOADELEMENTFAILEDPARSE2; } int * ct = (int *)hash_lookup(curelement, "list"); if (listid && (!ct || *ct <= 0)) goto LOADELEMENTFAILEDPARSE2; ctx->line = nil; int hydret = lhf(ctx); if ( 0 == hydret ){ fprint(2, "Node constructed as void. Saving will not include node: above line %s \n", line); } if ( -1 == hydret ){ LOADELEMENTFAILEDPARSE2: ; fprint(2, "file[%s] Failed to parse entry: above line %s \n", types[i]+SLTOFST, ""); } hash_reset (curelement); reset_temp_pool(); } return 1; } 役顔 * slovload(命一 **ring, uint * ringcur, 命三 ** 三ring){ //structure // [pwd/] // |-personaA.md // |-personaX.md // |-distil.md // |-head.id (contains GUID for current "most recent" set of memories) // |= Memories/ // |=GUIDX/ // ... // |=GUIDA/ // |-役顔.slov // |-連中.slov // |-命一.slov // |-命三.slov // |-vwrap.slov // |-残滓.slov // |-欠片.slov // |-ring.id //minimal setup includes head.id (with UID) distil.md and Memories/UID/役顔.slov which defines the user persona. char *memoryfilefmt = "./Memories/%s/%s.slov"; char *ringfilefmt = "./Memories/%s/ring.id"; // char *listitemfmt = "list_%0l5d"; // char *listitemkeybuffer = get_temp_string(15); char * types[] = {"void役顔", "void連中", "void命一", "void命三","voidvwrap", "void残滓", "void欠片"}; // 遺物箱 * mainpools[] = {役顔箱, 連中箱, 命一箱, 命三箱, vwrap箱, 残滓箱, 欠片箱}; int numtypes = TNUMTYPES; HashTable * curelement = hash_create(POOLSIZESML); HashTable * precommitdict = hash_create(POOLSIZEMAX); createvoidobjects(precommitdict); char * pass1役顔[] = {"血", "性", "地"};///, "名s" list char * pass1連中[] = {nil}; //for array size list char * pass1命一[] = {"token", "time", "役"}; char * pass1命三[] = {"一", "二", "三", "現役"}; //for array size char * pass1vec[] = {"v", "d"}; char * pass1残滓[] = {"欠片count", "声", "面影", "重", "三", "役", "vector", "d"}; char * pass1欠片[] = {"役", "type", "statement", "重", "三", "の", "vector", "d"}; char ** pass1[] = {pass1役顔, pass1連中, pass1命一, pass1命三, pass1vec, pass1残滓, pass1欠片}; int pass1s[] = { 3, 0, 3, 4, 2, 8, 8 }; char * pass1lists[] = {"名s", "現役", "面影", "何", nil, "結晶", nil}; char * pass2役顔[] = {nil}; char * pass2連中[] = {nil}; char * pass2命一[] = {"token", "time", "役","before", "after", "atbot", "atmid", "attop"}; char * pass2命三[] = {nil}; char * pass2vec[] = {nil}; char * pass2残滓[] = {nil}; char * pass2欠片[] = {nil}; char ** pass2[] = {pass2役顔, pass2連中, pass2命一, pass2命三, pass2vec, pass2残滓, pass2欠片}; int pass2s[] = { 0, 0, 8, 0, 0, 0, 0 }; char * pass2lists[TNUMTYPES] = {nil, nil, "面影", nil, nil, nil, nil}; Biobuf * head = Bopen("./head.id",OREAD); if(!head){ exits("no ./head.id unable to load memories"); } char *id = Brdstr(head, '\n', 1); //id needs freed at end! Biobuf * bios[TNUMTYPES] = {nil}; for(int i = 0; i < numtypes ; i++){ char *path = get_temp_string(0); int size = sprint(path, memoryfilefmt, id, types[i]+SLTOFST); get_temp_string(size+1); bios[i] = Bopen(path, OREAD); } Biobuf * ringbio; { char *path = get_temp_string(0); int size = sprint(path, ringfilefmt, id); get_temp_string(size+1); ringbio = Bopen(path, OREAD); } //no longer need head or id ringfilefmt slovloadctx ctx = (struct slovloadctx){id, bios, precommitdict, curelement, 0, pass1, pass1s, types, pass1lists, nil, loaderhydrate}; for(int i = 0; i < numtypes; i++){ ctx.i = i; if ( slovloadstatemachine(&ctx) < 0 ) break; } int processedcount = ctx.i; //pass 1 complete. because we read the types in a specific order we only need to fix up 命一 in pass 2 // 命一 has self referencing and references 命三 which is read later. // the order is defined to be minimal repeating of processing, because 命一 there's no avoiding doing a second pass for it. if (processedcount > 2){ Bseek(bios[2], 0, 0); ctx.i=2; ctx.items=pass2; ctx.itemscnt=pass2s; ctx.listids=pass2lists; ctx.line=nil; ctx.lhf=loaderstitch; if (bios[2]) slovloadstatemachine(&ctx); } //pass 2 complete. //we now need to move all objects read into the real hashmaps //as well as construct the synthetic renshuu "all" and load all the persona hnsw graphs 役顔 * distil = nil; 連中 * all中; {//create an "all" renshuu and commit each yaku //why no explicit check for "user" persona (血 == 1)? if you want 5 llm personas to just endlessly talk, go for it. int numberofitems = 役顔箱->num; 役顔 ** All = get遺物A(役顔p箱, 32); int allcount = 0; 役顔 * yakulist = (役顔 *)役顔箱->h; for ( int i = 0 ; i < numberofitems ; i++){ char * curname = yakulist[i].名; if(hash_lookup(precommitdict, curname) == &(yakulist[i])){ //it was commited with it's name and pointer, so it needs added to the real hashtable //yaku with missing 血 still in the system so their memories aren't culled, but we don't populate graphs hash_insert(HT_NAME_PERSONA, curname, &(yakulist[i])); //add persona to "all" if the chi is -1, 0, or 1 if ( (yakulist[i].血 >= -1) && (yakulist[i].血 <= 1) ){ if (allcount<32){ All[allcount++] = &(yakulist[i]); if (yakulist[i].血 >= 0){ //has memory yakulist[i].魂 = hnswcreate(50000, 10000, 50, 100, .6, nodebundledist); } for (int j = 0; j < yakulist[i].名sn; ++j){ if ( yakulist[i].名s[j] ) //making sure (hydrate has nil path) hash_insert(HT_NAME_PERSONA, yakulist[i].名s[j], &(yakulist[i])); } }else{ fprint (2, "Too many personas : %s cannot be switched to.\n", curname); //still in memories, just not currently avalibile. //reordering the personas so they are in the first 32 will make them loadable again and their history will be preserved. } } if (( yakulist[i].血 == -2 )){ // first distil persona wins if (distil == nil) distil = &(yakulist[i]); else fprint(2, "Found additional distilling persona %s. %s is being used. use /distil %s to switch\n", curname, distil->名, curname); } } } if (allcount < 1 ) { //need to have at least 1 persona ( if not there's probably some other problem ) fprint(2, "No personas found!\n"); exits("No personas"); } if (!distil){ //need to have a distil persona //the distil persona has strict schema requirements but it's unenforcable on our end except to reject improper distil output. fprint(2, "No Distil found!\n"); exits("No distil"); } all中 = get遺物(連中箱); all中->uid = find_add_string("ALL"); all中->現役数 = allcount; all中->現役 = All; hash_insert(HT_RENSHUU, all中->uid, all中); // fprint(2, "added ALL renshuu with %d personas\n", allcount); } {//commit each yaku including synthetic yaku int numberofitems = 連中箱->num; 連中 * renshuulist = (連中 *) 連中箱->h; for ( int i = 0; i < numberofitems; i++){ char * curguid = renshuulist[i].uid; //fprint(2, "Commiting yaku in renshuu (%s) - ", curguid); if(hash_lookup(precommitdict, curguid) == &(renshuulist[i])){ //fprint(2, "p matches LU - "); //it was commited with it's name and pointer, so it needs added to the real hashtable //usual else would be if the name was tied to void skip that name, the pointers are stitched. //This will capture /ALL/ yaku including the void and distil. This is correct and good. hash_insert(HT_RENSHUU, curguid, &(renshuulist[i])); //fprint(2, "generate key [%s]", generate連中key(&(renshuulist[i]),0) ); char * key = find_add_string(generate連中key(&(renshuulist[i]), 0)); hash_insert(HT_RENSHUU, key, &(renshuulist[i])); } //fprint(2, "\n"); } } // i don't think the other types need "committed" i think we just need to add all the loadable memories to the graphs { 残滓* residuelist = (残滓*) 残滓箱->h; int numberofitems = 残滓箱->num; for(int i = 0 ; i < numberofitems ; ++i){ char * curguid = residuelist[i].uid; if ( curguid == find_add_string(types[TGINSHI])) continue; //the actual void object if(hash_lookup(precommitdict, curguid) == &(residuelist[i]) ){ //the following makes sure this wasn't commited as void AND it makes sure the yaku is one of the 32 present personas. 役顔 * yaku = hash_lookup(HT_NAME_PERSONA, residuelist[i].役->名); if((yaku == residuelist[i].役) && yaku->魂) { if(!(residuelist[i].amb == 1)) continue; hnswinsert(yaku->魂, &(residuelist[i])); //fprint(2, "inserted [%s] into %s's graph\n", residuelist[i].面影, yaku->名); for(int j = 0; j < residuelist[i].欠片count; ++j){ if(residuelist[i].結晶[j].amb == 2)//make sure it's valid { //fprint(2, "inserted [%s] into %s's graph\n", residuelist[i].結晶[j].statement, yaku->名); hnswinsert(yaku->魂, &(residuelist[i].結晶[j])); } } } } } } 命一 * void一 = hash_lookup(precommitdict, types[TINOICHI]); 命三 * void三 = hash_lookup(precommitdict, types[TINOSAN]); *ringcur = 0; //ends load pointing at first empty slot, not last filled. if (ringbio){ char * line; uint i = 0; while ( line = Brdstr(ringbio, '\n', 1) ){ 命一 *一 = (命一 *)hash_lookup(precommitdict, line); if ( 一 ){ ring[i & RINGMASK] = 一; i++; } free(line); }//set ring1==atbot of lastfilled and ring2==atmid of last filled : ring0 nil or same as ring1 and is to be set during loop *ringcur = i; for(int j=i;j=1){ 三ring[2] = ring[(i - 1) & RINGMASK]->atmid; 三ring[0] = 三ring[1] = ring[(i - 1) & RINGMASK]->atbot; } else { for (int i = 0; i < 3 ; ++i){ 三ring[i] = void三; } } } else { for (int i = 0; i < RINGSIZE; ++i){ ring[i] = void一; } for (int i = 0; i < 3 ; ++i){ 三ring[i] = void三; } } //term openbios if (ringbio) Bterm(ringbio); if (head) Bterm(head); for(int i = 0; i < numtypes ; i++){ if(bios[i]) Bterm(bios[i]); } hash_free(curelement); hash_free(precommitdict); return distil; }