/* === File: muffet/muffet.c === */
#include <u.h>
#include <libc.h>
#include <avl.h>
#include <bio.h>
#include <regexp.h>
#include "utils.h"
#include "muffet.h"

void
usage(void)
{
	exits("usage");
} 
	char* HTTPSBASE = "http://www.cybersavior.dev/";
	char* MDBASE	= "http://www.cybersavior.dev/md/";
	char* GEMBASE  = "gemini://cybersavior.dev/";
	char* AUTHOR = "Takarabune";
	char* TITLE = "cybersavior(7)";

	char* HTMLFOOTER = "\n<footer class='bottom'>\n\t<p><a href='./license.html' title='copydown'><img src='../media/identity/down.png' alt='copydown' height='31' width='31'></a>&emsp;&emsp;&emsp;&emsp;&emsp;Fragment of <a href='./denpa_webring.html' title='Denpa Webring'>The Denpa Webring</a>&emsp;&emsp;&emsp;&emsp;&emsp;<a href='./myugii_fannan.html'><img src='../media/img/services/myugii.gif' alt='myugii.cybersavior.dev' height='31' width='88'></a></p>\n\t<p><a href='./plan9.html'><img src='../media/identity/poweredby.png' alt='Powered by 計画IX' height='31'></a></p>\n</footer>\n</body>\n</html>\n";

char * irefrx ="([^ ^	]*) *(.*)";
char * irefrxfull ="([^ ^	]*) *([^;]*) *; *(.*)";
char * erefrx ="[#^]([^	 ]*) *([^ 	]*) *(.*)";
char * urlrx ="[hH][tT][Tt][pP][sS]?://([!-~]+) *(.*)?";
char * gemrx ="[gG][eE][mM][iI][nN][iI]://([!-~]+) *(.*)?";
char * namerx = "^([^	 ].*)";
char * eofrx = "^[ 	]*!";
char * valrx = "^[ 	](.*)";
char * listrx ="^[ 	][ 	](.*)";
char * linkrx ="^[ 	]?[	 ]([^:]*) : [hH][Tt]+[pP][sS]?://([!-~]+) *(.*)?";

Reprog * irefrp;
Reprog * irefrpfull;
Reprog * erefrp; 
Reprog * urlrp;
Reprog * gemrp;
Reprog * namerp;
Reprog * eofrp;
Reprog * valrp;
Reprog * listrp;
Reprog * linkrp;

Avltree *tree;

char * muffet_type_str[]= {"hub", "article", "text", "tag", "portfolio", "album", "literal", "feed", "other", "ERROR"};

muffet_type
getmuffettype(char * s){
		for(int i = 0; i < (int)o_ERROR; i++)
			if(mcuistrcmp(s, muffet_type_str[i]) == 0)
				return (muffet_type)i;
	return o_ERROR;
}

char * muffet_reftype_str[]={"intern", "img", "snd", "vid", "file", "txt", "html", "list", "archive", "yt", "http", "gemini", "bandcamp", "proto", "ERROR"};

muffet_reftype
getreftype(char * s){
		for(int i = 0; i < (int)r_ERROR; i++)
			if(mcuistrcmp(s, muffet_reftype_str[i]) == 0)
				return (muffet_reftype)i;
	return r_ERROR;
}

void
printmuffetinc(Avl *n, Biobuf *, int out, int){
	fprint(out, "\t\t%s\n", ((muffet_inc_ref*)n)->name);
}

void
printmuffetref(Avl *n, Biobuf *, int out, int){
	muffet_out_ref * oR = (muffet_out_ref*) n;
	fprint(out, "\t\t[%d]%s%s%s\n", oR->index, oR->desc?" - ":"", oR->desc?oR->desc:"", oR->name);
}

void
printmuffettag(Avl *n, Biobuf *, int out, int){
	fprint(out, "\t\t%s\n", ((muffettag*)n)->name);
}

void
printmuffetlink(Avl *n, Biobuf *, int out, int){
	muffetlink *L = (muffetlink*) n;
	fprint(out, "\t\t%s : %s\n", L->name, L->url);
}

void
printmuffetchild(Avl *n, Biobuf *, int out, int){
	fprint(out, "\t\t%s\n", ((muffetchild*)n)->name);
}

void
printrlisthtml(muffetnode * N, int fd, char * title){
		muffetlink* L = (muffetlink*)avlmin(N->links);
		if(L){
			fprint(fd, "\n\t<h3>%s</h3>\n", title);
			do{
				fprint(fd, "<a href='http://%s' title='%s'>%s</a></br>\n",
					L->url, L->desc?L->desc:"", L->name );
			}while(L = (muffetlink*)avlnext(L));
			fprint(fd, "<br>\n");
		}
}

void
printmuffetbody(Avl *n, Biobuf *, int out, int){
	muffetbody *C = (muffetbody*) n;
	if(C->refcount == 0){
		fprint(out, "\t\t%s\n", C->line);
	} else {
		fprint(out, "\t\t");
		for(int i = 0; i < C->refcount; ++i){
			fprint(out, "%s{%s}", C->parts[i],C->refs[i]);
		}
		fprint(out, "%s\n",C->parts[C->refcount]);
	}
}
//used specifically for the synthetic tagrank and feed pages which do not use direct db information, rather calculate information based on db contents.
//these pages do not have "children" and may not even have real db entries
void
html_header(int fd_html, char* displayname, char * cannonname, char * cannonname_alt) {

	if (cannonname_alt == nil){
		cannonname_alt = cannonname;
	}
	fprint(fd_html, "<!DOCTYPE html> \n<html lang='en'>\n<head>\n");
	fprint(fd_html, "<meta charset='UTF-8'>\n");
	fprint(fd_html, "\t<link rel='stylesheet' type='text/css' href='../links/main.css'>\n");
	fprint(fd_html, "\t<link rel='alternate' type='application/rss+xml' href='../links/rss.xml' title='RSS Feed'>\n");
	fprint(fd_html, "\t<link rel='icon' type='image/x-icon' href='../media/identity/favicon.ico'>\n");
	fprint(fd_html, "\t<title>%s - %s</title>\n", TITLE, displayname);
	if(bgexists(cannonname) > 0) {
		fprint(fd_html, "\t<style>html{background-image: url('../media/bg/%s.png'); z-index: -10; -o-background-size: cover; background-size: cover;}</style>\n", cannonname);
	}

	{
		fprint(fd_html, "</head>\n\n<body>\n\n<div class='menu'>\n\t<header><a href='home.html'><img src='../media/identity/banner.png' alt='%s' height='50'></a></header>\n\n\t<nav class='top'>\n", TITLE);
		muffetnode key = {.name=cannonname_alt};

		muffetnode *F, *H, *G;
		char * cannon;

		F = (muffetnode*)avllookup(tree, &key, 0);
				
		if(F && F->host){
			key.name = F->host;
			H = (muffetnode*)avllookup(tree, &key, 0);
			if(H){
				key.name = H->host;
				G = (H->host) ? (muffetnode*)avllookup(tree, &key, 0) : nil;
				if(G){
					muffetchild* C = (muffetchild*)avlmin(G->children);
					fprint(fd_html, "\t\t<ul class='up'>\n");
					do{
						cannon = cannonizename(C->name);
						char * bref = nil;
						if(C->node && C->node->bref)
							bref = htmlstring(C->node->bref);
						int b = mcuistrcmp(C->name, H->name) == 0;
						fprint(fd_html, "\t\t\t<li>%s<a href='%s.html' title='%s'>%s</a>%s</li>\n",
							b ? "<b><u>" : "", cannon, bref?bref:C->name, C->name, b ? "</u></b>" : "");
						free(cannon);
						if(bref)
							free(bref);
					}while(C = (muffetchild*)avlnext(C));
					fprint(fd_html, "\t\t</ul>\n\t</nav>\n");
				}else{
					cannon = cannonizename(H->name);
					char * bref = nil;
					if(H->bref)
						bref = htmlstring(H->bref);
					fprint(fd_html, "\t\t<ul class='up'>\n\t\t\t<li><b><a href='%s.html' title='%s'>%s</a></b></li>\n\t\t</ul>\n\t</nav>\n",
						 cannon, bref?bref:H->name, H->name);
					free(cannon);
					if(bref)
						free(bref);
				}
				fprint(fd_html, "</div>\n\n<div class='container'>\n\t<nav class='left'>\n\t\t<ul class='sib'>\n");
				muffetchild* C = (muffetchild*)avlmin(H->children);
				//we know C has to exist, since node is one of them
				do{
					char * bref = nil;
					if(C->node && C->node->bref)
						bref = htmlstring(C->node->bref);
					cannon = cannonizename(C->name);
					if(mcuistrcmp(C->name,cannonname) == 0){ //mcuistrcmp(C->name, F->name) == 0
						fprint(fd_html, "\t\t\t<li><b><a href='%s.html' title='%s'>\&gt;%s</a></b></li>\n",
								cannon, bref?bref:C->name, C->name);
					} else {
						fprint(fd_html, "\t\t\t<li><a href='%s.html' title='%s'>%s</a></li>\n",
								cannon, bref?bref:C->name, C->name);
					}
					free(cannon);
					if(bref)
						free(bref);
				} while(C = (muffetchild*)avlnext(C));
				fprint(fd_html, "\t\t</ul>\n\t</nav>\n");
			}
		}
	}
}

void
printmuffetbodyHTML(Avl *n, int fd){
	muffetnode* N = (muffetnode*)n;
	muffetbody* B = (muffetbody*)avlmin(N->bodylines);
	char * cannon;
	if(B){
		int s = 1;
		do{
			fprint(fd, "\n\t\t<p>");
			int i = 0;
			do{
				fprint(fd, "%s", B->parts[i]);
				if(i >= (B->refcount)){
					i = i+1;
					continue;
				}
				muffet_out_ref  k = {.index=s};
				muffet_out_ref* r = (muffet_out_ref*)avllookup(N->out_refs, &k, 0);
				if(!r){
					i = i+1;
					continue;
				}
				cannon = cannonizename(r->name);
				char * rdesc = nil;
				char * rtext = nil;
				if(r->desc)
					rdesc = htmlstring(r->desc);
				if(r->text)
					rtext = htmlstring(r->text);
				switch(r->type){
					case r_INTERN:
							fprint(fd, "<a href='%s.html' title='%s'>[%s]</a>",
							cannon, rdesc?rdesc:r->name, rtext?rtext:rdesc?rdesc:r->name);
						break;
					case r_IMG:
							fprint(fd, "<img max-height='500' src='%smedia/img/%s' alt='%s'>",
							HTTPSBASE, cannon, rdesc?rdesc:r->name);
						break;
					case r_SOUND:
							fprint(fd, "<audio controls><source src='%smedia/sound/%s.mp3' type='audio/mpeg'><a href='%smedia/sound/%s.mp3'>MP3</a></audio>", 
							HTTPSBASE, cannon, HTTPSBASE, cannon);
						break;
					case r_VIDEO:
							fprint(fd, "<video height='600' controls><source src='%smedia/video/%s.mp4 type='video/mp4'><source src='%smedia/video/%s.mov' type='video/mp4'><a href='%smedia/video/%s.mov'>[MOV]</a> <a href='%smedia/video/%s.mp4'>[mp4]</a></video>",
							HTTPSBASE, cannon, HTTPSBASE, cannon, HTTPSBASE, cannon, HTTPSBASE, cannon);
						break;
					case r_FILE:
							fprint(fd, "<a href='%s%s' title='%s'>[%s]</a>",
							HTTPSBASE, cannon, r->name, rdesc?rdesc:r->name);
						break;
					case r_TXT:
							fprint(fd, "<a href='%smedia/txt/%s' title='%s'>[%s]</a>",
							HTTPSBASE, cannon, rdesc?rdesc:r->name, r->name);
						break;
					case r_HTML:
							fprint(fd, "<a href='%smedia/html/%s' title='%s'>[%s]</a>",
							HTTPSBASE, cannon, rdesc?rdesc:r->name, r->name);
						break;
					case r_LIST:
							//find the avl item for the list
						{
							muffetnode key = {.name = decannonizename(r->name)};
							muffetnode *M = (muffetnode*)avllookup(tree, &key, 0);
							printrlisthtml(M, fd, key.name);
							free(key.name);
						}
						break;
					case r_ARCHIVE:
							fprint(fd, "<iframe src='https://archive.org/embed/%s' width='640' height='480' frameborder='0' allowfullscreen> %s - %s </iframe>",
							r->name, rdesc?rdesc:r->name, r->name);
						break;
					case r_YOUTUBE:
							fprint(fd, "<a href='http://www.youtube.com/watch?v=%s' title='%s'>[%s]</a>",
							r->name, rdesc?rdesc:r->name, r->name);
						break;
					case r_HTTP:
							fprint(fd, "<a href='http://%s' title='%s'>[%s]</a>",
							r->name, rdesc?rdesc:r->name, r->name);
						break;
					case r_GEMINI:
							fprint(fd, "[gemini://%s%s%s]", r->name, r->desc?" - ":"", r->desc?r->desc:r->name);
						break;
					case r_BANDCAMP:
							fprint(fd, "<iframe style='height:50px;' src='https://bandcamp.com/EmbeddedPlayer/album=%s' seamless> %s - %s </iframe>",
							r->name, rdesc?rdesc:r->name, r->name);
						break;
					case r_PROTOCOL:
					case r_ERROR:
					default:
						break;
				}
				free(cannon);
				if(rtext)
					free(rtext);
				if(rdesc)
					free(rdesc);
				s++; i++;
			}while(i <= B->refcount);
			fprint(fd, "</p>\n");
		}while(B = (muffetbody*)avlnext(B));
	}
}

void
printrlistgem(muffetnode * N, int fd){
	muffetlink* L = (muffetlink*)avlmin(N->links);
	if(L){
		do{
			fprint(fd, "=>http://%s %s - %s\n",
					L->url, L->name, L->desc?L->desc:"" );
		}while(L = (muffetlink*)avlnext(L));
	}
}

void
printmuffetbodyGEM(Avl *n, int fd){
	muffetnode* N = (muffetnode*)n;
	muffetbody* B = (muffetbody*)avlmin(N->bodylines);
	char * cannon;
		if(B){
			int s = 1, e = 1;
			fprint(fd, "\n");
			do{
				e += B->refcount;
				for(int i = 0; i < B->refcount; ++i){
					muffet_out_ref k = {.index=s+i};
					muffet_out_ref* r = (muffet_out_ref*)avllookup(N->out_refs, &k, 0);
					fprint(fd, "%s%s[%s]", B->parts[i], r->text?r->text:r->desc?r->desc:r->name, B->refs[i]);
				}
				fprint(fd,"%s\n", B->parts[B->refcount]);
				for(; s<e; ++s){
					muffet_out_ref k = {.index=s};
					muffet_out_ref* r = (muffet_out_ref*)avllookup(N->out_refs, &k, 0);
					cannon = cannonizename(r->name);
					switch(r->type){
						case r_INTERN:
			fprint(fd, "=>%s%s.gmi [%d] %s\n",
								GEMBASE, cannon, s, r->desc?r->desc:r->name);
									break;
						case r_IMG:
			fprint(fd, "=>%smedia/img/%s [%d] %s [image]\n",
								GEMBASE, cannon, s, r->desc?r->desc:r->name);
									break;
						case r_SOUND:
			fprint(fd, "=>%smedia/sound/%s.mp3 [%d] %s.mp3 [audio]\n",
								GEMBASE, cannon, s, r->desc?r->desc:r->name);
									break;
						case r_VIDEO:
			fprint(fd, "=>%smedia/video/%s [%d] %s [video]\n",
								GEMBASE, cannon, s, r->desc?r->desc:r->name);
									break;
						case r_FILE:
			fprint(fd, "=>%s%s [%d] %s [binary]\n",
								GEMBASE, cannon, s, r->desc?r->desc:r->name);
									break;
						case r_TXT:
			fprint(fd, "=>%smedia/txt/%s [%d] %s [txt]\n",
								GEMBASE, cannon, s, r->desc?r->desc:r->name);
									break;
						case r_HTML:
			fprint(fd, "=>%smedia/html/%s [%d] %s [html]\n",
								GEMBASE, cannon, s, r->desc?r->desc:r->name);
									break;
						case r_LIST:
						{
							muffetnode key = {.name = decannonizename(r->name)};
							muffetnode *M = (muffetnode*)avllookup(tree, &key, 0);
							printrlistgem(M, fd);
							free(key.name);
						}
					break;
						case r_ARCHIVE:
			fprint(fd, "=>http://archive.org/details/%s [%d] %s [archive.org]\n",
								r->name, s,  r->desc?r->desc:"");
									break;
						case r_YOUTUBE:
			fprint(fd, "=>http://www.youtube.com/watch?v=%s [%d] %s [youtube.com]\n",
								r->name, s,  r->desc?r->desc:"");
									break;
						case r_HTTP:
			fprint(fd, "=>http://%s [%d] %s\n", r->name, s,  r->desc?r->desc:"");
									break;
						case r_GEMINI:
			fprint(fd, "=>gemini://%s [%d] %s\n", r->name, s,  r->desc?r->desc:r->name);
									break;
						case r_BANDCAMP:
			fprint(fd, "=>https://bandcamp.com/EmbeddedPlayer/album=%s [%d] %s [bandcamp]",
								r->name, s,  r->desc?r->desc:"");
						case r_PROTOCOL:
						case r_ERROR:
						default:
								break;
					}

					free(cannon);
				}
			}while(B = (muffetbody*)avlnext(B));
		}
}


void
printmuffetnode(Avl *n, Biobuf *in, int out, int dir){
	muffetnode * N = (muffetnode*) n;
	int fd = 0;
	if(N->host){
		if(N->type == o_TEXT){
			 fd = opentext(N->name);
		}
		fprint(out, "\n%s\n", N->name);
		fprint(out, "\tHOST : %s\n", N->host);
		fprint(out, "\tBREF : %s\n", N->bref);
		fprint(out, "\tTYPE : %s\n", muffet_type_str[(int)N->type]);
		if(avlmax(N->children)){
			fprint(out, "\tCHILDREN\n");
			walk(N->children, in, out, printmuffetchild, dir);
		}
		if(avlmax(N->bodylines)){
			fprint(out, "\tBODY\n");
			walk(N->bodylines, in, out, printmuffetbody, 0);
		}
		if(avlmax(N->links)){
			fprint(out, "\tLINK\n");
			walk(N->links, in, out, printmuffetlink, 0);
		}
		if(avlmin(N->tags)){
			fprint(out, "\tTAGS\n");
			walk(N->tags, in, out, printmuffettag, dir);
		}
		if(avlmin(N->inc_refs)){
			fprint(out, "\tINC\n");
			walk(N->inc_refs, in, out, printmuffetinc, dir);
		}
		if(avlmin(N->out_refs)){
			fprint(out, "\tREF\n");
			walk(N->out_refs, in, out, printmuffetref, dir);
		}
		if(fd>0){
			Biobuf * txt = Bfdopen(fd, OREAD);
			char * line;
			while(line = Brdstr(txt, '\n', 0)){
				fprint(out, "%s", line);
			}
			fprint(out, ".\n");
			Bterm((Biobufhdr*) txt);
		}
	} else {
		fprint(out, "\n%s\n", N->name);
		if(avlmax(N->links)){
			fprint(out, "\tLINK\n");
			walk(N->links, in, out, printmuffetlink, 0);
		}
	}
}

char*
fetchslov(Biobuf *in, int out, char *name, char *key){
	char * line;
	char * ret = nil;
	fprint(out, "\nl\n%s\n\t%s\n!\n", name, key);
	line = Brdstr(in, '\n', 1);
	match[0].sp = match[0].ep = 0;
	if(regexec(valrp, line, match, MSZ)){
		ret = copymatch(1);
		free(line);
	} else {
		while(line = Brdstr(in, '\n', 1)){
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				free(line);
				break;
			}
			free(line);
		}
	}
	return ret;
}

void
writeFeed(Biobuf *in, int out){
	muffetnode key = {.name = "FEED"};
	muffetnode* F = (muffetnode*)avllookup(tree, &key, 0);
	char *linepage, *linepub, *linedate, *newpub, *newdate;
	int foundnew = 0;
	
	if (F) {
		int fd_rss  = opentemp("rss.xml");
		int fd_atom  = opentemp("atom.xml");
		int fd_gmi  = opentemp("feed.gmi");
		int fd_html  = opentemp("feed.html");

		
		fprint(fd_atom, "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n");
		fprint(fd_rss , "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n");
		fprint(fd_atom, "<feed xmlns=\"http://www.w3.org/2005/Atom\">\n\n");
		fprint(fd_rss , "<rss xmlns:dc=\"http://purl.org/dc/elements/1.1/\" version=\"2.0\">\n\n");
		fprint(fd_atom, "\t<title>%s</title>\n", TITLE);
		fprint(fd_rss , "\t<channel>\n\t<title>%s</title>\n", TITLE);
		fprint(fd_gmi , "#%s\n\n", TITLE);
		html_header(fd_html, "Feed", "feed", nil);
		fprint(fd_html, "\n\t<div class='main'>\n\t\t\n\t\t<h1>Feed</h1>\n\t\t\n");
		fprint(fd_atom, "\t<link href=\"%s\"/>\n", GEMBASE);
		fprint(fd_rss , "\t<link> %s/ </link>\n", HTTPSBASE);
		fprint(fd_rss , "\t<description> Cybersavior(7) </description>\n");

		fprint(out, "l\n%s\n\tDATE\n!\n", ((muffetchild*)avlmax(F->children))->node->name);
		linedate = Brdstr(in, '\n', 1);
		fprint(out, "l\n%s\n\tPUB\n!\n", ((muffetchild*)avlmax(F->children))->node->name);
		linepub = Brdstr(in, '\n', 1);
		fprint(out, "l\n%s\n\tNDATE\n!\n", ((muffetchild*)avlmax(F->children))->node->name);
		newdate = Brdstr(in, '\n', 1);
		fprint(out, "l\n%s\n\tNPUB\n!\n", ((muffetchild*)avlmax(F->children))->node->name);
		newpub = Brdstr(in, '\n', 1);
		
		match[0].sp = match[0].ep = 0; 
		if(regexec(valrp, linedate, match, MSZ))
			linedate = match[1].sp;

		match[0].sp = match[0].ep = 0;
		if(regexec(valrp, linepub, match, MSZ))
			linepub = match[1].sp;

		match[0].sp = match[0].ep = 0; 
		if(regexec(valrp, newdate, match, MSZ)){
			newdate = match[1].sp;
			foundnew++;
		}
		match[0].sp = match[0].ep = 0;
		if(regexec(valrp, newpub, match, MSZ)){
			newpub = match[1].sp;
			foundnew++;
		}
		if(foundnew < 1){
			fprint(fd_atom, "\t<updated>%sT12:00:00Z</updated>\n", linedate);
			fprint(fd_rss, "\t<lastBuildDate>%s 00:00:00 +0800</lastBuildDate>\n", linepub);
		} else {
			fprint(fd_atom, "\t<updated>%s</updated>\n", newdate);
			fprint(fd_rss,  "\t<lastBuildDate>%s</lastBuildDate>\n", newpub);
		}
		free(linedate);
		free(linepub);
		free(newdate);
		free(newpub);
	
		fprint(fd_atom, "\t<id>%s</id>\n\n", GEMBASE);
		//fprint(fd_rss , "\t<image> <url>%smedia/services/rss.png<\url> <\image>\n", HTTPSBASE);

		muffetchild* C = (muffetchild*)avlmax(F->children);
		do{
			muffetnode* c = C->node;
			char * escped = nil;
			if ( c->bref )
				escped = htmlstring(c->bref);
			fprint(fd_atom, "\t<entry>\n");
			fprint(fd_rss , "\t<item>\n");
			fprint(fd_atom, "\t\t<title>%s</title>\n", escped?escped:c->name);
			fprint(fd_atom, "\t\t<author><name>%s</name></author>\n", AUTHOR);
			fprint(fd_rss , "\t\t<title>%s</title>\n", escped?escped:c->name);
			fprint(fd_rss , "\t\t<dc:creator>%s</dc:creator>\n", AUTHOR);
			if (escped) { free(escped); escped = nil; }
		
			fprint(out, "l\n%s\n\tPAGE\n!\n", c->name);

			linepage = Brdstr(in, '\n', 1);
			match[0].sp = match[0].ep = 0; regexec(valrp, linepage, match, MSZ);
			linepage = match[1].sp;

			fprint(fd_atom, "\t\t<link href=\"%s%s.gmi\" />\n", GEMBASE, linepage);
			fprint(fd_rss , "\t\t<link>%ssite/%s.html</link>\n", HTTPSBASE, linepage);
			{
 				char * cannon;
 				cannon = cannonizename(c->name);
				fprint(fd_atom, "\t\t<id>\"%s%s.gmi\"</id>\n", GEMBASE, linepage);
				fprint(fd_rss , "\t\t<guid isPermaLink=\"false\">%ssite/%s.html</guid>\n", HTTPSBASE, cannon);
				free(cannon);
			}

			foundnew = 0;

			fprint(out, "l\n%s\n\tDATE\n!\n", c->name);
			linedate = Brdstr(in, '\n', 1);
			//fprint(2, "	linedate:%s \n", linedate);
			match[0].sp = match[0].ep = 0;
			if(regexec(valrp, linedate, match, MSZ))
				linedate = match[1].sp;

			fprint(out, "l\n%s\n\tPUB\n!\n", c->name);
			linepub = Brdstr(in, '\n', 1);
			//fprint(2, "	linepub:%s \n", linepub);
			match[0].sp = match[0].ep = 0;
			if(regexec(valrp, linepub, match, MSZ))
				linepub = match[1].sp;

			fprint(out, "l\n%s\n\tNDATE\n!\n", c->name);
			newdate = Brdstr(in, '\n', 1);
			//fprint(2, "	newdate:%s \n", newdate);
			match[0].sp = match[0].ep = 0;
			if(regexec(valrp, newdate, match, MSZ)){
				newdate = match[1].sp;
				foundnew++;
			}

			fprint(out, "l\n%s\n\tNPUB\n!\n", c->name);
			newpub = Brdstr(in, '\n', 1);
			match[0].sp = match[0].ep = 0;
			//fprint(2, "	newpub:%s \n", newpub);
			if(regexec(valrp, newpub, match, MSZ)){
				newpub = match[1].sp;
				foundnew++;
			}
			if(foundnew < 1){
				fprint(fd_atom, "\t\t<updated>%sT12:00:00Z</updated>\n", linedate);
				fprint(fd_rss , "\t\t<pubDate>%s 00:00:00 +0800</pubDate>\n", linepub);
			} else {
				fprint(fd_atom, "\t\t<updated>%s</updated>\n", newdate);
				fprint(fd_rss , "\t\t<pubDate>%s</pubDate>\n", newpub);
			}
			fprint(fd_gmi, "=>%s%s.gmi %s - %s\n", GEMBASE, linepage, linedate, c->bref);


			fprint(fd_html, "\t\t<p><a href='./%s.html'>%s - %s</a></p>\n", linepage, linedate, c->bref);
			free(newdate);
			free(newpub);
			free(linepage);
			free(linedate);
			muffetbody * body = (muffetbody*) avlmin(c->bodylines);
			if(body){
				fprint(fd_atom, "\t\t<summary>\n");
				fprint(fd_rss, "\t\t<description><![CDATA[\n");
				printmuffetbodyGEM(c, fd_atom);
				printmuffetbodyHTML(c, fd_rss);
				fprint(fd_atom, "\t\t</summary>\n");
				fprint(fd_rss, "\t\t]]></description>\n");
			}

			fprint(fd_atom, "\t</entry>\n\n");
			fprint(fd_rss, "\t</item>\n\n");
		}while(C = (muffetchild*)avlprev(C));
		fprint(fd_atom, "</feed>\n\n");
		fprint(fd_rss , "\t</channel>\n</rss>\n");
		fprint(fd_html, "\n\t<h1>Historical records</h1>\n\n\t<p>\n\t\t\tRSS feed <a href='%slinks/rss.xml' title='rss.xml'>[rss.xml]</a>\n\t\t</p>\n\t</div>\n\n</div>\n", HTTPSBASE);
		fprint(fd_html, "%s", HTMLFOOTER);
	
		close(fd_atom);
		close(fd_rss);
		close(fd_gmi);
		close(fd_html);
		

		checkandreplace("rss.xml", t_LINK);
		checkandreplace("atom.xml", t_LINK);
		checkandreplace("feed.gmi", t_GEM);
		checkandreplace("feed.html", t_HTML);
	}
}

int
comptagcountnode(void * A, void * B){ 
	tagcountnode *a, *b;
	a = (tagcountnode *)A;
	b = (tagcountnode *)B;
	return (b->count - a->count);
}






void
writeTagrank(void){
	tagcountnode* TAGcountlist;
	//annoyingly avltree doesn't keep track of how many elements are in the tree, so lets (sigh) count them
	int c = 0;
	muffetnode* N = (muffetnode*)avlmin(tree);
	if(N != nil){
		do{
			c++;
		}while(N = (muffetnode*)avlnext(N));
	}
	TAGcountlist = malloc(sizeof(tagcountnode)*c);
	int j=0;
	N = (muffetnode*)avlmin(tree);
	if(N != nil){
		do{
			TAGcountlist[j].name = N->name;
			TAGcountlist[j].count = 0;
			//now count the number of incoming references
			muffet_inc_ref * R = (muffet_inc_ref*)avlmin(N->inc_refs);
			if(R != nil){
				do{
					TAGcountlist[j].count++;
				}while(R = (muffet_inc_ref*)avlnext(R));
			}
			j++;
		}while(N = (muffetnode*)avlnext(N));
	}
	qsort(TAGcountlist, c, sizeof(TAGcountlist[0]), comptagcountnode);
	//fprint(2, "%d\n", c);
	for(int i=0; i<c; i++){
		//fprint(2, "%d	- %s \n", TAGcountlist[i].count, TAGcountlist[i].name);
	}
	if (c > 5 ) {
		int fd_gmi = opentemp("tagrank.gmi");
		int fd_html = opentemp("tagrank.html");

		fprint(fd_gmi , "#%s\n\n", TITLE);
		html_header(fd_html, "Tag Rank", "tagrank", "tags");

		fprint(fd_html, "\n\t<div class='main'>\n\t<h1>Tag Rank</h1>\n");
		for(int i=0; i<c; i++){
			int v = TAGcountlist[i].count;
			if (v >= 2){
				char * n = TAGcountlist[i].name;
				char * cannon = cannonizename(n);
				char * ws = "   ";
				fprint(fd_gmi, "=>%s%s.gmi %s%d - %s\n", GEMBASE, cannon, ws + ((v > 99)?2:((v > 9)?1:0)), v, n);
				fprint(fd_html, "\t\t<p><a href='./%s.html'>%s%d - %s</a></p>\n", cannon, ws + ((v > 99)?2:((v > 9)?1:0)), v, n);
				free(cannon);
			}
		}

		fprint(fd_html, "\t</div>\n\n</div>\n" );
		
		fprint(fd_html, "%s", HTMLFOOTER);

		close(fd_gmi);
		close(fd_html);

		checkandreplace("tagrank.gmi", t_GEM);
		checkandreplace("tagrank.html",t_HTML);
	}
	
}

void
doHTML(void){
	muffetnode* N = (muffetnode*)avlmin(tree);
	char filename[100];
	do{
		if(N->type == o_ERROR )
			continue;
		if(mcuistrcmp(N->name, "feed") == 0)
			continue;
		if(mcuistrcmp(N->name, "tagrank") == 0)
			continue;

		char * cannon;
		cannon = cannonizename(N->name);
		snprint(filename, 100, "%s.html",cannon);
		int fd = opentemp(filename);

		fprint(fd, "<!DOCTYPE html> \n<html lang='en'>\n<head>\n");
		fprint(fd, "<meta charset='UTF-8'>\n");
		fprint(fd, "\t<link rel='stylesheet' type='text/css' href='../links/main.css'>\n");
		fprint(fd, "\t<link rel='alternate' type='application/rss+xml' href='../links/rss.xml' title='RSS Feed'>\n");
		fprint(fd, "\t<link rel='icon' type='image/x-icon' href='../media/identity/favicon.ico'>\n");
		fprint(fd, "\t<title>%s - %s</title>\n", TITLE, N->name);
			

		if(bgexists(cannon) > 0) 
			fprint(fd, "\t<style>html{background-image: url('../media/bg/%s.png'); z-index: -10; -o-background-size: cover; background-size: cover;}</style>\n", cannon);
		free(cannon);
		fprint(fd, "</head>\n\n<body>\n");
		fprint(fd, "<div class='menu'>\n");
		fprint(fd, "\t<header><a href='home.html'><img src='../media/identity/banner.png' alt='%s' height='50'></a></header>\n\t<nav class='top'>\n", TITLE);
			
		muffetnode key;
		muffetnode* F;

		//parent and it's siblings
		if(N->host){
			fprint(fd, "\t\t<ul class='up'>\n");
				//find host's host.
		
			key.name = N->host;
			F = (muffetnode*)avllookup(tree, &key, 0);
			if(F && F->host){
				key.name = F->host;
				F = (muffetnode*)avllookup(tree, &key, 0);
				if(F){ 
					muffetchild* C = (muffetchild*)avlmin(F->children);
					do{
						cannon = cannonizename(C->name);
						char * cbref = nil;
						if(C->node && C->node->bref)
							cbref = htmlstring(C->node->bref);
						int b = mcuistrcmp(C->name, N->host) == 0;
						fprint(fd, "\t\t\t<li>%s<a href='%s.html' title='%s'>%s</a>%s</li>\n",
							b?"<b><u>":"", cannon, cbref?cbref:C->name, C->name, b?"</u></b>":"");
						free(cannon);
						if(cbref)
							free(cbref);
					}while(C = (muffetchild*)avlnext(C));
				
				}
			}else if(F){
				cannon = cannonizename(F->name);
				char * fbref = nil;
				if(F->bref)
					fbref = htmlstring(F->bref);
				fprint(fd, "\t\t\t<li><b><u><a href='%s.html' title='%s'>%s</a></u></b></li>\n",
						cannon,fbref?fbref:F->name, F->name);
				free(cannon);
				if(fbref) free(fbref);
			}
			fprint(fd, "\t\t</ul>\n\t</nav>\n</div>\n\n");
		} else {			//home and various list nodes have no host. They still need to follow some kind of proper conventions
							//most especially home as it is the site root node.
			cannon = cannonizename(N->name);
			char * nbref = nil;
			if(N->bref)
				nbref = htmlstring(N->bref);
			fprint(fd, "\t\t<ul class='up'>\n");
			fprint(fd, "\t\t\t<li><b><u><a href='%s.html' title='%s'>\&gt;%s</a></u></b></li>\n",
						cannon, nbref?nbref:N->name, N->name);
			fprint(fd, "\t\t</ul>\n\t</nav>\n</div>\n\n");
			free(cannon);
			if(nbref) free(nbref);
		}

			//siblings and item
		if(N->host){
			fprint(fd, "<div class='container'>\n\t<nav class='left'>\n\t\t<ul class='sib'>\n");
				//find host.
			key.name = N->host;
			F = (muffetnode*)avllookup(tree, &key, 0);
			if(F){
				muffetchild* C = (muffetchild*)avlmin(F->children);
				//we know C has to exist, since node is one of them
				do{
					cannon = cannonizename(C->name);
					char * cbref = nil;
					if(C->node && C->node->bref)
						cbref = htmlstring(C->node->bref);
					if(mcuistrcmp(C->name, N->name) == 0){
				 		fprint(fd, "\t\t\t<li><u><b><a href='%s.html' title='%s'>\&gt;%s</a></u></b></li>\n",
							cannon, cbref?cbref:C->name, C->name);
//children
						
						muffetchild* Z = (muffetchild*)avlmin(N->children);
						if(Z){
							fprint(fd, "\n\n\t\t\t<ul class='child'>\n");
							do{
								char *cannonZ = cannonizename(Z->name);
								char *zbref = nil;
								if(Z->node && Z->node->bref)
									zbref = htmlstring(Z->node->bref);
								fprint(fd, "\t\t\t\t<li>\&emsp;<a href='%s.html' title='%s'>%s</a></li>\n",
									cannonZ, zbref?zbref:Z->name, Z->name);
								free(cannonZ);
								if(zbref) free(zbref);
							}while(Z = (muffetchild*)avlnext(Z));
							fprint(fd, "\t\t\t</ul>\n\n");
						}
					} else {
						fprint(fd, "\t\t\t<li><a href='%s.html' title='%s'>%s</a></li>\n",
							cannon, cbref?cbref:C->name, C->name);
					}
					free(cannon);
					if(cbref) free(cbref);
				}while(C = (muffetchild*)avlnext(C));
			}
			fprint(fd, "\t\t</ul>\n\t</nav>\n\n");
		} else { //again special case for home (and other hostless nodes)
			fprint(fd, "<div class='container'>\n\t<nav class='left'>\n\t\t<ul class='sib'>\n");
			key.name = N->name;
			F = (muffetnode*)avllookup(tree, &key, 0);
			if(F){
				muffetchild* C = (muffetchild*)avlmin(F->children);
				if(C){
					do{
						cannon = cannonizename(C->name);
						char *cbref = nil;
						if(C->node && C->node->bref)
							cbref = htmlstring(C->node->bref);
						fprint(fd, "\t\t\t<li><a href='%s.html' title='%s'>%s</a></li>\n",
								cannon, cbref?cbref:C->name, C->name);
						free(cannon);
						if(cbref) free(cbref);
					}while(C = (muffetchild*)avlnext(C));
				}
			}
			fprint(fd, "\t\t</ul>\n\t</nav>\n\n");
			
		}

		//body
		fprint(fd, "\t<div class='main'>\n\t\t<h1>%s</h1>\n", N->name);
		if(N->bref){
			fprint(fd, "\t\t<h2> - %s - </h2>\n",N->bref);
		}
		fprint(fd, "\t\t\n");

		printmuffetbodyHTML(N, fd);

		if(N->type == o_TEXT){
			fprint(fd, "\n\t\t<p style='white-space: pre-wrap'>");
			int tfd = opentext(N->name);
			if(tfd>0){
				Biobuf * txt = Bfdopen(tfd, OREAD);
				char * line;
				while(line = Brdstr(txt, '\n', 0)){
					fprint(fd, "%s", line);
				}
				fprint(fd, "\n");
			}
			close(tfd);
			fprint(fd, "\t\t</p>\n");
		}

	//tags
		muffettag* T = (muffettag*)avlmin(N->tags);
		if(T){
			fprint(fd, "\n\t\t<h2>tags</h2>\n\t\t<p>\n");
			do{
				cannon = cannonizename(T->name);
				char* bref = nil;
				if(T->node && T->node->bref)
					bref = htmlstring(T->node->bref);
				fprint(fd, "\t\t\t<a href='%s.html' title='%s'>%s</a>\n",
						cannon, bref?bref:" ", T->name );
				free(cannon);
				if(bref) free(bref);
			}while(T = (muffettag*)avlnext(T));
			fprint(fd, "\t\t</p>\n");
		}

	//links
		muffetlink* L = (muffetlink*)avlmin(N->links);
		if(L){
			fprint(fd, "\n\t\t<h2>additional links</h2>\n\t\t<p>");
			do{
				fprint(fd, "\t\t\t<a href='http://%s' title='%s'>%s</a>\n",
					L->url, L->desc?L->desc:"", L->name );
			}while(L = (muffetlink*)avlnext(L));
			fprint(fd, "\t\t</p>\n");
		}
	//refs
		muffet_inc_ref* iR = (muffet_inc_ref*)avlmin(N->inc_refs);
		if(iR){
			fprint(fd, "\n\t\t<h2>incoming references</h2>\n\t\t<p>");
			do{
				cannon = cannonizename(iR->name);
				char* bref = nil;
				if(iR->node && iR->node->bref)
					bref = htmlstring(iR->node->bref);
				fprint(fd, "\t\t\t<a href='%s.html' title='%s'>%s</a>\n",
						cannon, bref?bref:"", iR->name);
				free(cannon);
				if(bref) free(bref);
			}while(iR = (muffet_inc_ref*)avlnext(iR));
			fprint(fd, "\t\t</p>\n");
		}
		
		fprint(fd, "\t</div>\n\n</div>\n");
		fprint(fd, "%s", HTMLFOOTER);
		close(fd);
		checkandreplace(filename, t_HTML);
	}while(N = (muffetnode*)avlnext(N));
}



void
doGemini(void){
	
	muffetnode* N = (muffetnode*)avlmin(tree);
	char filename[100];
	do{	
		if(N->type == o_ERROR )
			continue;
		if(mcuistrcmp(N->name, "feed") == 0)
			continue;
		if(mcuistrcmp(N->name, "tagrank") == 0)
			continue;

		char * cannon;
		cannon = cannonizename(N->name);
		snprint(filename, 100, "%s.gmi", cannon);
		int fd = opentemp(filename);
		free(cannon);
		if(N->host){
			cannon = cannonizename(N->host);
			fprint(fd, "###HOME\n=>home.gmi HOME\n\n");
			fprint(fd, "###host\n");
			fprint(fd, "=>%s%s.gmi %s\n", GEMBASE, cannon, N->host);
			free(cannon);
			cannon = cannonizename(N->name);
			if(bgexists(cannon) > 0) 
				fprint(fd, "=>%smedia/bg/%s.png background[image]\n", GEMBASE, cannon);
			free(cannon);
		}
		muffetchild* C = (muffetchild*)avlmin(N->children);
		if(C){
			fprint(fd, "###children\n");
			do{
				if(C->node->type <= o_OTHER){
					cannon = cannonizename(C->name);
					fprint(fd, "=>%s%s.gmi %s - %s\n",
							GEMBASE, cannon, C->name, C->node->bref);
					free(cannon);
				}
			}while(C = (muffetchild*)avlnext(C));
		}

		muffettag* T = (muffettag*)avlmin(N->tags);
		if(T){
			fprint(fd,"\n###tags\n");
			do{
				cannon = cannonizename(T->name);
				fprint(fd, "=>%s%s.gmi #%s - %s\n",
						GEMBASE, cannon, T->name, T->node?T->node->bref:" " );
				free(cannon);
			}while(T = (muffettag*)avlnext(T));
		}


		fprint(fd, "\n\n\n#%s\n", N->name);
		if(N->bref) fprint(fd, "##%s\n", N->bref);

		printmuffetbodyGEM(N, fd);

		if(N->type == o_TEXT){
			int tfd = opentext(N->name);
			if(tfd>0){
				Biobuf * txt = Bfdopen(tfd, OREAD);
				char * line;
				while(line = Brdstr(txt, '\n', 0)){
					fprint(fd, "%s", line);
				}
				fprint(fd, ".\n");
			}
			close(tfd);
		}


		muffetlink* L = (muffetlink*)avlmin(N->links);
		if(L){
			fprint(fd, "\n###additional links\n");
			do{
				fprint(fd, "=>http://%s %s - %s\n",
						L->url, L->name, L->desc?L->desc:"" );
			}while(L = (muffetlink*)avlnext(L));
		}
		
		muffet_inc_ref* iR = (muffet_inc_ref*)avlmin(N->inc_refs);
		if(iR){
			fprint(fd, "\n###incoming references\n");
			do{
				cannon = cannonizename(iR->name);
				fprint(fd, "=>%s%s.gmi %s - %s\n", GEMBASE, cannon, iR->name, iR->node?(iR->node->bref?iR->node->bref:""):"");
				free(cannon);
			}while(iR = (muffet_inc_ref*)avlnext(iR));
		}
		
		close(fd);

		checkandreplace(filename, t_GEM);
		
	}while(N = (muffetnode*)avlnext(N));
}

void
doMD(void){
	
	muffetnode* N = (muffetnode*)avlmin(tree);
	char filename[100];
	do{	
		if(N->type == o_ERROR )
			continue;
		if(mcuistrcmp(N->name, "feed") == 0)
			continue;
		if(mcuistrcmp(N->name, "tagrank") == 0)
			continue;

		char * cannon;
		cannon = cannonizename(N->name);
		snprint(filename, 100, "%s.md", cannon);
		int fd = opentemp(filename);
		free(cannon);
		if(N->host){
			cannon = cannonizename(N->host);
			fprint(fd, "# [HOME](home.md)\n\n");
			fprint(fd, "HOST: [%s](%s%s.md)  \n",N->host, MDBASE, cannon );
			free(cannon);
			cannon = cannonizename(N->name);
			if(bgexists(cannon) > 0) { ////HERE
				fprint(fd, "![BG IMG](%smedia/bg/%s.png)", HTTPSBASE, cannon);
				fprint(fd, "=>%smedia/bg/%s.png background[image]\n", GEMBASE, cannon);
			}
			free(cannon);
		}
		muffetchild* C = (muffetchild*)avlmin(N->children);
		if(C){
			fprint(fd, "###children\n");
			do{
				if(C->node->type <= o_OTHER){
					cannon = cannonizename(C->name);
					fprint(fd, "=>%s%s.gmi %s - %s\n",
							GEMBASE, cannon, C->name, C->node->bref);
					free(cannon);
				}
			}while(C = (muffetchild*)avlnext(C));
		}

		muffettag* T = (muffettag*)avlmin(N->tags);
		if(T){
			fprint(fd,"\n###tags\n");
			do{
				cannon = cannonizename(T->name);
				fprint(fd, "=>%s%s.gmi #%s - %s\n",
						GEMBASE, cannon, T->name, T->node?T->node->bref:" " );
				free(cannon);
			}while(T = (muffettag*)avlnext(T));
		}


		fprint(fd, "\n\n\n#%s\n", N->name);
		if(N->bref) fprint(fd, "##%s\n", N->bref);

		printmuffetbodyGEM(N, fd);

		if(N->type == o_TEXT){
			int tfd = opentext(N->name);
			if(tfd>0){
				Biobuf * txt = Bfdopen(tfd, OREAD);
				char * line;
				while(line = Brdstr(txt, '\n', 0)){
					fprint(fd, "%s", line);
				}
				fprint(fd, ".\n");
			}
			close(tfd);
		}


		muffetlink* L = (muffetlink*)avlmin(N->links);
		if(L){
			fprint(fd, "\n###additional links\n");
			do{
				fprint(fd, "=>http://%s %s - %s\n",
						L->url, L->name, L->desc?L->desc:"" );
			}while(L = (muffetlink*)avlnext(L));
		}
		
		muffet_inc_ref* iR = (muffet_inc_ref*)avlmin(N->inc_refs);
		if(iR){
			fprint(fd, "\n###incoming references\n");
			do{
				cannon = cannonizename(iR->name);
				fprint(fd, "=>%s%s.gmi %s - %s\n", GEMBASE, cannon, iR->name, iR->node?(iR->node->bref?iR->node->bref:""):"");
				free(cannon);
			}while(iR = (muffet_inc_ref*)avlnext(iR));
		}
		
		close(fd);

		checkandreplace(filename, t_GEM);
		
	}while(N = (muffetnode*)avlnext(N));
}
void
freemuffetnode(Avl * n){
	muffetnode		* N = (muffetnode *) n;
	muffetchild		* C;
	muffetbody		* B;
	muffetlink		* L;
	muffet_inc_ref	* iR;
	muffet_out_ref	* oR;
	muffettag		* T;
	
	muffetchild Ckey	 =	{ .name = ""};
	muffetbody Bkey		 =	{ .index = 0};
	muffetlink Lkey 	 =	{ .index = 0};
	muffet_inc_ref iRkey =	{ .name = ""};
	muffet_out_ref oRkey =	{ .index = 0};
	muffettag Tkey		 =	{ .name = ""};

	free(N->bref);
	free(N->host);
	free(N->name);

	while(C = (muffetchild*) avllookup(N->children, &Ckey, -1)){
		C = (muffetchild*) avldelete(N->children, C);
		freemuffetchild(C);
	}
	while(B = (muffetbody*) avllookup(N->bodylines, &Bkey, 1)){
		B = (muffetbody*) avldelete(N->bodylines, B);
		freemuffetbody(B);
	}
	while(L = (muffetlink*) avllookup(N->links, &Lkey, 1)){
		L = (muffetlink*) avldelete(N->links, L);
		freemuffetlink(L);
	}
	while(iR = (muffet_inc_ref*) avllookup(N->inc_refs, &iRkey, -1)){
		iR = (muffet_inc_ref*) avldelete(N->inc_refs, iR);
		freemuffet_inc_ref(iR);
	}
	while(oR = (muffet_out_ref*) avllookup(N->out_refs, &oRkey, 1)){
		oR = (muffet_out_ref*) avldelete(N->out_refs, oR);
		freemuffetchild(oR);
	}
	while(T = (muffettag*) avllookup(N->tags, &Tkey, -1)){
		T = (muffettag*) avldelete(N->tags, T);
		freemuffettag(T);
	}

	free(N->children);
	free(N->bodylines);
	free(N->links);
	free(N->inc_refs);
	free(N->out_refs);
	free(N->tags);
	free(N);
}
void
freemuffetchild (Avl * n){
	muffetchild	*N = (muffetchild *) n;
	//free(N->name); these belong to the child
	//free(N->node);
	free(N);
}
void
freemuffetbody (Avl * n){
	muffetbody	*N = (muffetbody *) n;
	free(N->line);
	free(N->refs);
	free(N->parts);
	free(N);
}
void
freemuffetlink (Avl * n){
	muffetlink	*N = (muffetlink *) n;
	free(N->name);
	free(N->url);
	free(N);
}
void
freemuffet_inc_ref (Avl * n){
	muffet_inc_ref	*N = (muffet_inc_ref *) n;
	//free(N->name); belongs to the node
	//don't free the node
	free(N);
}
void
freemuffet_out_ref (Avl * n){
	muffet_inc_ref	*N = (muffet_inc_ref *) n;
	//free(N->name);
	//free(N->desc); //these belong to body's line
	free(N);
}
void
freemuffettag (Avl * n){
	muffet_inc_ref	*N = (muffet_inc_ref *) n;
	free(N->name);
	free(N);
}




void
populateNAME(Biobuf * in, int out){
	muffetnode * recentmn;
	Avl * dup;

	char * line;

	fprint(out, "l\n!\n");
	while( line = Brdstr(in, '\n', 1)){
		match[0].sp = match[0].ep = 0;
		if(regexec(eofrp, line, match, MSZ)){
		//	print("got to end\n");
			free(line);
			break;
		}
	
		match[0].sp = match[0].ep = 0;
		if(regexec(namerp, line, match, MSZ)){
			recentmn = (muffetnode *)malloc(sizeof(muffetnode));
			recentmn->name = copymatch(1);
			recentmn->host = nil;
			recentmn->bref = nil;
			recentmn->type = o_OTHER;
			recentmn->children = avlcreate(mccmp);
			recentmn->bodylines = avlcreate(mbcmp);
			recentmn->links = avlcreate(mlcmp);
			recentmn->inc_refs = avlcreate(mircmp);
			recentmn->out_refs = avlcreate(morcmp);
			recentmn->tags = avlcreate(mtcmp);
			if(dup = avlinsert(tree, recentmn)){
				freemuffetnode(dup);
			}
		}
		free(line);
	}
}
/*
void populateALL(Avl * n, Biobuf *in, int out, int){
	muffetnode * N = (muffetnode *) n;
	muffetnode * H;
	muffetnode Hkey;
	muffetchild * NaC;
	char * rK, *V;

	char * line;
	char* allLUstr = "\nl\n%s\n!\n";

	fprint(out, allLUstr, N->name);
	line = Brdstr(in, '\n', 1);
	match[0].sp = match[0].ep = 0;
	if(regexec(valrp, line, match, MSZ)){
		N->host = copymatch(1);
		Hkey.name = N->host;
		H = (muffetnode*)avllookup(tree, &Hkey, 0);
		if(H){
			NaC = (muffetchild *)malloc(sizeof(muffetchild));
			NaC->name = N->name;
			NaC->node = N;
			NaC = (muffetchild *)avlinsert(H->children, NaC);
			if(NaC)
				freemuffetchild(NaC);
		}
		free(line);
	} else { 
		free(line);
	}
}

void
populateNAMEnew(Biobuf * in, int out){
	muffetnode * recentmn;
	Avl * dup;

	char * line;

	fprint(out, "l\n!\n");
	while( line = Brdstr(in, '\n', 1)){
		match[0].sp = match[0].ep = 0;
		if(regexec(eofrp, line, match, MSZ)){
		//	print("got to end\n");
			free(line);
			break;
		}
	
		match[0].sp = match[0].ep = 0;
		if(regexec(namerp, line, match, MSZ)){
			recentmn = (muffetnode *)malloc(sizeof(muffetnode));
			recentmn->name = copymatch(1);
			recentmn->host = nil;
			recentmn->bref = nil;
			recentmn->type = o_OTHER;
			recentmn->children = avlcreate(mccmp);
			recentmn->bodylines = avlcreate(mbcmp);
			recentmn->links = avlcreate(mlcmp);
			recentmn->inc_refs = avlcreate(mircmp);
			recentmn->out_refs = avlcreate(morcmp);
			recentmn->tags = avlcreate(mtcmp);
			if(dup = avlinsert(tree, recentmn)){
				freemuffetnode(dup);
			}
		}
		free(line);
	}

}
*/


void
populateHOST(Avl * n, Biobuf *in, int out, int){
	muffetnode * N = (muffetnode *) n;
	muffetnode * H;
	muffetnode Hkey;
	muffetchild * NaC;

	char * line;
	char* hostlustr = "\nl\n%s\n\tHOST\n!\n";

	fprint(out, hostlustr, N->name);
	line = Brdstr(in, '\n', 1);
	match[0].sp = match[0].ep = 0;
	if(regexec(valrp, line, match, MSZ)){
		N->host = copymatch(1);
		Hkey.name = N->host;
		H = (muffetnode*)avllookup(tree, &Hkey, 0);
		if(H){
			NaC = (muffetchild *)malloc(sizeof(muffetchild));
			NaC->name = N->name;
			NaC->node = N;
			NaC = (muffetchild *)avlinsert(H->children, NaC);
			if(NaC)
				freemuffetchild(NaC);
		}
		free(line);
	} else { 
		free(line);
	}
}

void indexPrint(muffetnode* N , int out, int depth){
	char *cannon = cannonizename(N->name);
	fprint(out, "		{%s |", cannon);
	free(cannon);
	for(int i = 1; i < depth; ++i) fprint(out, "==");	
	fprint(out, "%s}\n", N->name);
	muffetchild * current = (muffetchild*)avlmin(N->children);
	if(current && current->node != N){
		do{
			indexPrint(current->node, out, depth+1);
			
		}while(current = (muffetchild*)avlnext(current));
	}
}

void buildINDEX( int out){
	fprint(out, "\ninsert\nindex\n	BODY\n");
	muffetnode hkey = {.name="home"};
	muffetnode* home = (muffetnode*)avllookup(tree, &hkey,0);
	indexPrint(home, out, 1);

	fprint(out,"\n!\n");
}

void populateBREF(Avl * n, Biobuf *in, int out, int ){
	muffetnode * N = (muffetnode *) n;

	char * line;
	char* breflustr = "\nl\n%s\n\tBREF\n!\n";

	fprint(out, breflustr, N->name);
	line = Brdstr(in, '\n', 1);
	match[0].sp = match[0].ep = 0;
	if(regexec(valrp, line, match, MSZ)){
		N->bref = copymatch(1);
		free(line);
	} else { 
		 free(line);
	}
}

void populateTYPE(Avl * n, Biobuf *in, int out, int ){
	muffetnode * N = (muffetnode *) n;

	char * line;
	char* typelustr = "\nl\n%s\n\tTYPE\n!\n";
	char * tmatch;

	fprint(out, typelustr, N->name);
	line = Brdstr(in, '\n', 1);
	match[0].sp = match[0].ep = 0;
	if(regexec(valrp, line, match, MSZ)){
		tmatch = copymatch(1);
		N->type = getmuffettype(tmatch);
		free(tmatch);
		free(line);
	} else { 
		free(line);
	}
}

#define REFSPERLINE 20
Avl * 
fillbodynode(char * line, muffet_type type, int bodycount){
	char * parts[REFSPERLINE+1];
	char * refs[REFSPERLINE];
	muffetbody * B = (muffetbody*) malloc(sizeof(muffetbody));
	B->line = line;
	B->index = bodycount;
	B->refs = nil;
	B->parts = nil;
	int refcount = 0;	

	if(type == o_LITERAL){
	//literal type disables meta processing
		B->refcount = refcount;
		return B;
	}
	parts[0] = line;
	
	for(int i = 0; line[i] != 0; ++i){
		if(line[i] == '{'){
			line[i] = 0;
			refs[refcount++] = line+i+1;
		}
		if(line[i] == '}'){
			line[i] = 0;
			parts[refcount] = line+i+1;
		}
	}
	B->refs  = (char**) malloc(sizeof(char*) *  refcount);
	B->parts = (char**) malloc(sizeof(char*) * (refcount + 1));
	B->refcount = refcount;
	for(int i = 0; i < refcount; ++i)
		B->refs[i] = refs[i];
	for(int i = 0; i <= refcount; ++i)
		B->parts[i] = parts[i];
	return B;
}

void
populateBODY(Avl * n, Biobuf *in, int out, int ){
	muffetnode * N = (muffetnode *) n;
	muffetbody * B = nil;

	char * line;
	char* bodylustr = "\nl\n%s\n\tBODY\n!\n";
	char * tmatch;
	int bodycount = 0;
	int firstline = 1;
	int nobody = 0;

	
	fprint(out, bodylustr, N->name);
	while (line = Brdstr(in, '\n', 1)){
//	fprint (2, " %s\n", line);
		if(firstline){
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				//the last line will be eof (!)
				free(line);
				break;
			}
			firstline = 0;
			match[0].sp = match[0].ep = 0;
			if(regexec(namerp, line, match, MSZ)){
				//we found a host name, not BODY
				nobody = 1;
				free(line);
				continue; 
			}
			match[0].sp = match[0].ep = 0;
			if(regexec(valrp, line, match, MSZ)){
				tmatch = copymatch(1);
				if(mcuistrcmp(tmatch,"BODY") == 0){
					free(tmatch);
					free(line);
					continue;
				} else { //single line body
					match[0].sp = match[0].ep = 0;
					if(regexec(valrp, line, match, MSZ)){
						tmatch = copymatch(1);
						B = (muffetbody*) fillbodynode(tmatch, N->type, ++bodycount);
						B = (muffetbody*) avlinsert(N->bodylines, B);
					}
					if(B)
						freemuffetbody(B);
					free(line);
					break;
				}
			}
		}
		if(nobody){
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				//the last line will be eof (!)
				free(line);
				break;
			}
			free(line);
		} else {
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				//the last line will be eof (!)
				free(line);
				break;
			}
			match[0].sp = match[0].ep = 0;
			if(regexec(listrp, line, match, MSZ)){
				tmatch = copymatch(1);
				B = (muffetbody*) fillbodynode(tmatch, N->type, ++bodycount);
				B = (muffetbody*) avlinsert(N->bodylines, B);
				if(B)
					freemuffetbody(B);
				free(line);
				continue; 
			}
			free(line);
		}
	}
}

void
populateLINK(Avl * n, Biobuf *in, int out, int ){
	muffetnode * N = (muffetnode *) n;
	muffetlink * L ;

	char * line;
	char* linklustr = "\nl\n%s\n\tLINK\n!\n";
	char* urllustr  = "\nl\n%s\n\t%s\n!\n";
	char* keylustr  = "\nl\n%s\n\tKEY\n!\n";
	// we added special keyname "key" which will return the names of all the keys
	// this way we can populate links in the form:
	/*********************************************************
	A LIST OF LINKS
		some site name : some site url
		another site name : another url
		etc. : etc.
	*********************************************************/
	//additionally we added a keyname 'list' 
	//this will get all the list names for lists which are of the form:
	/*********************************************************
	A LIST OF ITEMS
		something
		another thing
		and yet something else
	*********************************************************/
	//we just have to make sure we ignore the "well known" names (ie host, link, etc.)
	char * tmatch;
	int linkcount = 0;
	int firstline = 1;
	int nolink = 0;

	
	fprint(out, linklustr, N->name);
	while (line = Brdstr(in, '\n', 1)){
	//print ("%s\n", line);
		if(firstline){
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				//the last line will be eof (!)
				free(line);
				break;
			}
			firstline = 0;
			match[0].sp = match[0].ep = 0;
			if(regexec(namerp, line, match, MSZ)){
				//we found a host name, not LINK
				nolink = 1;
				free(line);
				continue; 
			}
			match[0].sp = match[0].ep = 0;
			if(regexec(valrp, line, match, MSZ)){
				tmatch = copymatch(1);
				if(mcuistrcmp(tmatch,"LINK") == 0){
					free(tmatch);
					free(line);
					continue;
				} else { //single line link
					match[0].sp = match[0].ep = 0;
					if(regexec(linkrp, line, match, MSZ)){
						L = (muffetlink*) malloc(sizeof(muffetlink));
						L->index = ++linkcount;
						L->name = copymatch(1);
						L->url = copymatch(2);
						L->desc = copymatch(3);
						L = (muffetlink*) avlinsert(N->links,L);
						if(L)
							freemuffetlink(L);
					}
					free(line);
					break;
				}
			}
		}
		if(nolink){
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				//the last line will be eof (!)
				free(line);
				break;
			}
			free(line);
		} else {
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				//the last line will be eof (!)
				free(line);
				break;
			}
			match[0].sp = match[0].ep = 0;
			if(regexec(linkrp, line, match, MSZ)){
				L = (muffetlink*) malloc(sizeof(muffetlink));
				L->index = ++linkcount;
				L->name = copymatch(1);
				L->url = copymatch(2);
				L->desc = copymatch(3);
				L = (muffetlink*) avlinsert(N->links,L);
				if(L)
					freemuffetlink(L);
			}
			free(line);
		}
	}
	fprint(out, keylustr, N->name);
	while (line = Brdstr(in, '\n', 1)){
		if (mcuistrcmp(line, "!") == 0 ){
			free(line);
			break;
		}
		if (mcuistrcmp(line, "	body") == 0 ||
			mcuistrcmp(line, "	link") == 0 ||
			mcuistrcmp(line, "	tags") == 0 ||
			mcuistrcmp(line, "	bref") == 0 ||
			mcuistrcmp(line, "	host") == 0 ||
			mcuistrcmp(line, "	date") == 0 ||
			mcuistrcmp(line, "	page") == 0 ||
			mcuistrcmp(line, "	pub") == 0 ||
			mcuistrcmp(line, "	ndate") == 0 ||
			mcuistrcmp(line, "	npub") == 0 ||
			mcuistrcmp(line, "	type") == 0){
				free(line);
				continue;
		}
		match[0].sp = match[0].ep = 0;
		if(regexec(valrp, line, match, MSZ)){
			L = (muffetlink*) malloc(sizeof(muffetlink));
			L->index = ++linkcount;
			L->name = copymatch(1);
			//fprint(2, "%s-%s-\n",N->name, L->name);
			L->url = nil;
			L->desc = nil;
			L = (muffetlink*) avlinsert(N->links,L);
			if(L)
				freemuffetlink(L);
		}
	}

	L = (muffetlink*)avlmax(N->links);
	if(L){
		do{
			if(L && L->url == nil){
				fprint(out, urllustr, N->name, L->name);
				line = Brdstr(in, '\n', 1);
				match[0].sp = match[0].ep = 0;
				if(regexec(urlrp, line, match, MSZ) ){
					L->url = copymatch(1);
					L->desc = copymatch(2);
				}
				
			}
		}while (L = (muffetlink*)avlprev(L));
	}
}

void
populateTAG(Avl * n, Biobuf *in, int out, int ){
	muffetnode * N = (muffetnode *) n;
	muffettag * T;

	char * line;
	char* taglustr = "\nl\n%s\n\tTAGS\n!\n";
	char * tmatch;
	int firstline = 1;
	int notag = 0;

	
	fprint(out, taglustr, N->name);
	while (line = Brdstr(in, '\n', 1)){
	//print ("%s\n", line);
		if(firstline){
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				//the last line will be eof (!)
				free(line);
				break;
			}
			firstline = 0;
			match[0].sp = match[0].ep = 0;
			if(regexec(namerp, line, match, MSZ)){
				notag = 1;
				free(line);
				continue; 
			}
			match[0].sp = match[0].ep = 0;
			if(regexec(valrp, line, match, MSZ)){
				tmatch = copymatch(1);
				if(mcuistrcmp(tmatch,"TAGS") == 0){
					free(tmatch);
					free(line);
					continue;
				} else { //single line link
					match[0].sp = match[0].ep = 0;
					if(regexec(valrp, line, match, MSZ)){
						T = (muffettag*) malloc(sizeof(muffettag));
						T->name = copymatch(1);
						T->node = (muffetnode*) avllookup(tree, T, 0);
						if(T->node == nil){
							fprint(2, "Tag node %s not found\n", T->name);
							freemuffettag(T);
							free(line);
							continue;
						}
						muffet_inc_ref * TaR = (muffet_inc_ref *)malloc(sizeof(muffet_inc_ref));
						TaR->name = N->name;
						TaR->node = N;
						TaR = (muffet_inc_ref*) avlinsert(T->node->inc_refs, TaR);
						//fprint(2, "put %s into %s \n",T->name,  T->node->name); 
						if(TaR)
							freemuffet_inc_ref(TaR);
						T = (muffettag*) avlinsert(N->tags, T);
						if(T)
							freemuffettag(T);
					}
					free(line);
					break;
				}
			}
		}
		if(notag){
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				//the last line will be eof (!)
				free(line);
				break;
			}
			free(line);
		} else {
			match[0].sp = match[0].ep = 0;
			if(regexec(eofrp, line, match, MSZ)){
				//the last line will be eof (!)
				free(line);
				break;
			}
			match[0].sp = match[0].ep = 0;
			if(regexec(listrp, line, match, MSZ)){
				//fprint(2, "got here: %s \n", line);
				T = (muffettag*) malloc(sizeof(muffettag));
				T->name = copymatch(1);
				T->node = (muffetnode*) avllookup(tree, T, 0);
				if(T->node == nil){
					fprint(2, "Tag node %s not found\n", T->name);
					freemuffettag(T);
					free(line);
					continue;
				}
				muffet_inc_ref * TaR = (muffet_inc_ref *)malloc(sizeof(muffet_inc_ref));
				TaR->name = N->name;
				TaR->node = N;
				TaR = (muffet_inc_ref*) avlinsert(T->node->inc_refs, TaR);
				//fprint(2, "put %s into %s \n",N->name,  T->node->name); 
				if(TaR)
					freemuffet_inc_ref(TaR);
				T = (muffettag*) avlinsert(N->tags, T);
				if(T)
					freemuffetlink(T);
				
			}
			free(line);
		}
	}
}

void
calculateREFS(Avl * n, Biobuf *, int, int ){
	muffetnode * N = (muffetnode*) n;
	muffetnode * tN;
	muffetnode tNkey;
	char * tstr;
	int refcount = 0;
	muffet_out_ref * oR;

	muffet_inc_ref * iR;

	muffetbody* B = (muffetbody*) avlmin(N->bodylines);
	do{
		for(int i = 0 ; B && i < B->refcount; ++i){
			match[0].sp = match[0].ep = 0;
			if(regexec(erefrp, B->refs[i], match, MSZ)){
				tstr = copymatch(1);
				oR = (muffet_out_ref*) malloc(sizeof(muffet_out_ref));
				oR->index=++refcount;
				oR->name = copymatch(2);
				oR->type = getreftype(tstr);
				oR->text = nil;
				oR->desc = copymatch(3);
				sprint(B->refs[i], "%d", refcount);
				oR = (muffet_out_ref*) avlinsert(N->out_refs, oR);
				if(oR)
					freemuffet_out_ref(oR);
				continue;
			}
			match[0].sp = match[0].ep = 0;
			if(regexec(urlrp,  B->refs[i], match, MSZ)){
				oR = (muffet_out_ref*) malloc(sizeof(muffet_out_ref));
				oR->index=++refcount;
				oR->name = copymatch(1);
				oR->type = getreftype("http");
				oR->text = nil;
				oR->desc = copymatch(2);
				sprint(B->refs[i], "%d", refcount);
				oR = (muffet_out_ref*) avlinsert(N->out_refs, oR);
				if(oR)
					freemuffet_out_ref(oR);
				continue;
			}
			if(regexec(gemrp,  B->refs[i], match, MSZ)){
				oR = (muffet_out_ref*) malloc(sizeof(muffet_out_ref));
				oR->index=++refcount;
				oR->name = copymatch(1);
				oR->type = getreftype("gemini");
				oR->text = nil;
				oR->desc = copymatch(2);
				sprint(B->refs[i], "%d", refcount);
				oR = (muffet_out_ref*) avlinsert(N->out_refs, oR);
				if(oR)
					freemuffet_out_ref(oR);
				continue;
			}

			//TODO : CHECK THIS WORKS : REVERT FROM ALT IF NOT
			match[0].sp = match[0].ep = 0;
			if(regexec(irefrpfull,  B->refs[i], match, MSZ)){
				oR = (muffet_out_ref*) malloc(sizeof(muffet_out_ref));
				oR->index=++refcount;
				oR->name = copymatch(1);
				oR->type = getreftype("intern");
				oR->text = copymatch(2);
				oR->desc = copymatch(3);
				sprint(B->refs[i], "%d", refcount);

				tNkey.name = oR->name;
				tN = (muffetnode*)avllookup(tree, &tNkey, 0);
				if(tN){
					iR = (muffet_inc_ref*) malloc(sizeof(muffet_inc_ref));
					iR->name = N->name;
					iR->node = N;
					iR = (muffet_inc_ref*) avlinsert(tN->inc_refs, iR);
					if(iR)
						freemuffet_inc_ref(iR);
				}
				
				oR = (muffet_out_ref*) avlinsert(N->out_refs, oR);
				if(oR)
					freemuffet_out_ref(oR);
	
			} else {
				match[0].sp = match[0].ep = 0;
				if(regexec(irefrp,  B->refs[i], match, MSZ)){
					oR = (muffet_out_ref*) malloc(sizeof(muffet_out_ref));
					oR->index=++refcount;
					oR->name = copymatch(1);
					oR->type = getreftype("intern");
					oR->text = copymatch(2);
					oR->desc = oR->name;
					sprint(B->refs[i], "%d", refcount);
	
					tNkey.name = oR->name;
					tN = (muffetnode*)avllookup(tree, &tNkey, 0);
					if(tN){
						iR = (muffet_inc_ref*) malloc(sizeof(muffet_inc_ref));
						iR->name = N->name;
						iR->node = N;
						iR = (muffet_inc_ref*) avlinsert(tN->inc_refs, iR);
						if(iR)
							freemuffet_inc_ref(iR);
					}
					
					oR = (muffet_out_ref*) avlinsert(N->out_refs, oR);
					if(oR)
						freemuffet_out_ref(oR);
		
				}
			}


		}
	}while(B && (B = (muffetbody*) avlnext(B)));
}



void
main( int , char *[] )
{

	Biobuf * slovdat;
	int slovcmd;
	//char *line;

	irefrp = regcomp(irefrx);
	irefrpfull = regcomp(irefrxfull);
	erefrp = regcomp(erefrx);
		
	valrp  = regcomp(valrx);
	eofrp  = regcomp(eofrx);
	namerp = regcomp(namerx);
	listrp = regcomp(listrx);
	linkrp = regcomp(linkrx);
	urlrp  = regcomp(urlrx);
	gemrp  = regcomp(gemrx);

	tree = avlcreate(mncmp);

	slovdat = Bopen("#s/slov.dat", OREAD);

	if(slovdat == nil){
		fprint(2, " slov.dat must be running! " );
		exits(nil);
	}
	
	slovcmd = open("#s/slov.cmd", OWRITE);

	if(slovcmd < 1){
		fprint(2, " slov.cmd must be running! " );
		exits(nil);
		
	}
	
	populateNAME(slovdat, slovcmd);
	
	walk(tree, slovdat, slovcmd, populateHOST, 1);
	buildINDEX(slovcmd);
	walk(tree, slovdat, slovcmd, populateBREF, 1);
	walk(tree, slovdat, slovcmd, populateTYPE, 1);
	walk(tree, slovdat, slovcmd, populateBODY, 1);
	walk(tree, slovdat, slovcmd, populateLINK, 1);
	walk(tree, slovdat, slovcmd, populateTAG, 1);
	walk(tree, slovdat, slovcmd, calculateREFS, 1); 
	walk(tree, nil, 1, printmuffetnode, 0);
	doGemini();
	doHTML();
	//doMD();
	writeTagrank();
	writeFeed(slovdat, slovcmd);


	free(irefrp);
	free(erefrp);
	free(irefrpfull);
	free(valrp);
	free(eofrp);
	free(namerp);
	free(listrp);
	free(linkrp);
	free(urlrp);
	free(gemrp);


}


/* === File: muffet/utils.c === */
#include <u.h>
#include <libc.h>
#include <avl.h>
#include <bio.h>
#include <regexp.h>
#include "utils.h"
#include "muffet.h"

char* OUTDIR = "/lib/muffet/";
char* OUTTMP = "/tmp/";


Resub match[MSZ];

char * 
htmlstring(char *s){
	int len = strlen(s);
	char *formatted = malloc((len*6)+1);
	char *end = formatted + ((len*6)+1);
	char *writehead = formatted;
	*writehead = 0;
	for(int i=0; s[i] != 0 ; i++){
		switch(s[i]){
			case '&':
				writehead = seprint(writehead, end, "%s", "&amp;");
				break;
			case '\'':
				writehead = seprint(writehead, end, "%s", "&#39;");
				break;
			case '\"':
				writehead = seprint(writehead, end, "%s", "&quot;");
				break;
			case '<':
				writehead = seprint(writehead, end, "%s", "&lt;");
				break;
			case '>':
				writehead = seprint(writehead, end, "%s", "&gt;");
				break;
			default:
				*writehead = s[i];
				writehead++;
				*writehead = 0;
				break;
		}
	}
	return formatted;
}


char *
cannonizename(char * name){
	//fprint(2, " %s  - %d\n", name, strlen(name));
	int len = strlen(name);
	char * cannonname = (char*) malloc(len+1);
	for(int i = 0; name[i] != 0 ; ++i){
		if(name[i] == ' '){
			cannonname[i] = '_';
		} else if(name[i] >= 'A' && name[i] <= 'Z'){
			cannonname[i] = name[i] + 32;
		} else {
			cannonname[i] = name[i];
		}
	}
	cannonname[len] = 0;
	return cannonname;
}
char *
decannonizename(char * name){
	//fprint(2, " %s  - %d\n", name, strlen(name));
	int len = strlen(name);
	char * cannonname = (char*) malloc(len+1);
	for(int i = 0; name[i] != 0 ; ++i){
		if(name[i] == '_'){
			cannonname[i] = ' ';
		} else if(name[i] >= 'a' && name[i] <= 'z'){
			cannonname[i] = name[i] - 32;
		} else {
			cannonname[i] = name[i];
		}
	}
	cannonname[len] = 0;
	return cannonname;
}

int
opentemp(char * name){
	char *pathfmt = "%s%s";
	char buf[100];
	int fd;
	name = cannonizename(name);
	snprint(buf, 100, pathfmt, OUTTMP, name);
	fd = create(buf, OWRITE, 0664);
	free(name);
	return(fd);
}

int
checkandreplace(char * name, tmptype t){
	char *gempathfmt  = "%sgemini/%s";
	char *htmlpathfmt = "%shtml/%s";
	char *linkpathfmt = "%slinks/%s";
	char *tmppathfmt  = "%s%s";
	int tfd;
	int rfd;
	char tbuf[100];
	char rbuf[100];
	char * rpathfmt = ((t==t_HTML)?(htmlpathfmt):((t==t_GEM)?gempathfmt:linkpathfmt));

	name = cannonizename(name);
	snprint(tbuf, 100, tmppathfmt, OUTTMP, name);
	tfd = open(tbuf, OREAD);
	snprint(rbuf, 100, rpathfmt, OUTDIR, name);
	rfd = open(rbuf, OREAD);
	
	Dir * td = dirfstat(tfd);
	Dir * rd = dirfstat(rfd);
	vlong tl = 0;
	if(td){
		tl = td->length;
	}
	vlong rl = 0;
	if(rd){
		rl = rd->length;
	}
	free(td);
	free(rd);

	int different = 0;
	if( rl != tl ){
		different = 1;
	}

	if(different == 0){
		Biobuf tbio;
		Biobuf rbio;
		Binit(&tbio, tfd, OREAD);
		Binit(&rbio, rfd, OREAD);
		int done = 0;
		while(different == 0 && !done){
			int tc = Bgetc(&tbio);
			int rc = Bgetc(&rbio);
			different =  tc - rc;
			done = (tc == Beof) || (rc == Beof);
		}
		Bterm(&tbio);
		Bterm(&rbio);
	}
	close(tfd);
	close(rfd);

	if(different != 0){
		Biobuf *tbio = Bopen(tbuf, OREAD);
		Biobuf *rbio = Bopen(rbuf, OWRITE | OTRUNC);
		int j;
		while((j = Bgetc(tbio)) != Beof){
			Bputc(rbio, j);
		}
		Bterm(tbio);
		Bterm(rbio);
		return(1);
	}
	return(0);
}

int
openfile_rel(char *name, char *fmt, int mode, int perm){
	char buf[100];
	char *c = cannonizename(name);
	snprint(buf, 100, fmt, OUTDIR, c);
	free(c);
	if(mode & OWRITE)
		return create(buf, mode, perm);
	return open(buf, mode);
}

int
opentext(char * name){
	return openfile_rel(name, "%smedia/txt/%s.txt", OREAD, 0);
}

int
openlinks(char * name){
	return openfile_rel(name, "%slinks/%s", OWRITE, 0664);
}

int
bgexists(char * name){
	char buf[100];
	snprint(buf, 100, "%smedia/bg/%s.png", OUTDIR, name);
	int t = open(buf, OREAD);
	close(t);
	return t;
}

int
opengem(char * name){
	return openfile_rel(name, "%sgemini/%s.gmi", OWRITE, 0664);
}

int
openhtml(char * name){
	return openfile_rel(name, "%shtml/%s.html", OWRITE, 0664);
}

char * 
copymatch(int i)
{
	int len;
	char * s_match;
	if(match[i].ep && match[i].sp && match[i].ep > match[i].sp){
		len = (long long)match[i].ep - (long long)match[i].sp;
		s_match = malloc(len+1);
		strncpy(s_match,match[i].sp, len);
		s_match[len] = 0;
		for(int i = len -1 ; s_match[i] == ' ' || s_match[i] == '	'; i--)
			s_match[i] = 0;
		return s_match;
	}
	return nil;
}

int
mcuistrcmp(char* a, char* b)
{
	int ca, cb;
	for(;;){
		ca = (*a == '_') ? ' ' : ((*a > 'Z' && *a < '{') ? (*a - 32) : *a);
		cb = (*b == '_') ? ' ' : ((*b > 'Z' && *b < '{') ? (*b - 32) : *b);
		if(ca != cb || ca == 0)
			return ca - cb;
		a++; b++;
	}
}

int
mncmp( Avl * i , Avl * j)
{
	return mcuistrcmp(((muffetnode*)i)->name, ((muffetnode*)j)->name);
}

int
mccmp( Avl * i , Avl * j)
{
	return mcuistrcmp(((muffetchild*)i)->name, ((muffetchild*)j)->name);
}

int
mbcmp( Avl * i , Avl * j)
{
	return ((muffetbody*)i)->index - ((muffetbody*)j)->index;
}

int
mlcmp( Avl * i , Avl * j)
{
	return ((muffetlink*)i)->index - ((muffetlink*)j)->index;
}

int
mircmp( Avl * i , Avl * j)
{
	return mcuistrcmp(((muffet_inc_ref*)i)->name, ((muffet_inc_ref*)j)->name);
}

int
morcmp( Avl * i , Avl * j)
{
	return ((muffet_out_ref*)i)->index - ((muffet_out_ref*)j)->index;
}

int
mtcmp( Avl * i , Avl * j)
{
	return mcuistrcmp(((muffettag*)i)->name, ((muffettag*)j)->name);
};

void
walk(Avltree * tree, Biobuf * in, int out, void(*action)(Avl *, Biobuf * in, int out, int dir), int dir){
	Avl * dot;
	if(dir){dot = avlmax(tree);} else {dot = avlmin(tree);}
	do{
		if(dot){
			action(dot, in, out, dir);
		}
	}while (dot = dir?avlprev(dot):avlnext(dot));
}


/* === File: muffet/muffet.h === */


typedef struct muffetnode muffetnode;
typedef struct muffetchild muffetchild;
typedef struct muffetbody muffetbody;
typedef struct muffetlink muffetlink;
typedef struct muffet_inc_ref muffet_inc_ref;
typedef struct muffet_out_ref muffet_out_ref;
typedef struct muffettag muffettag;


typedef enum muffet_type{
	o_HUB = 0,
	o_ARTICLE,
	o_TEXT,
	o_TAG,
	o_PORTFOLIO,
	o_ALBUM,
	o_LITERAL,
	o_FEED,
//	o_TEXTPORTAL, //depreciated -> articles
	o_OTHER,
	o_ERROR
}muffet_type;

typedef enum muffet_reftype{
	r_INTERN = 0,
	r_IMG,
	r_SOUND,
	r_VIDEO,
	r_FILE,
	r_TXT,
	r_HTML,
	r_LIST,
	r_ARCHIVE,
	r_YOUTUBE,
	r_HTTP,
	r_GEMINI,
	r_BANDCAMP,
	r_PROTOCOL,
	r_ERROR
}muffet_reftype;



/*
switch(r->type){
	case r_INTERN:
	case r_IMG:
	case r_SOUND:
	case r_VIDEO:
	case r_TXT:
	case r_HTML:
	case r_LIST:
	case r_ARCHIVE:
	case r_YOUTUBE:
	case r_HTTP:
	case r_BANDCAMP:
	case r_PROTOCOL:
	case r_ERROR:
	default:
}
*/

extern char * muffet_type_str[];
extern char * muffet_reftype_str[];

extern muffet_type getmuffettype(char *);
extern muffet_reftype getreftype(char *);

extern void printmuffetchild(Avl *n, Biobuf *in, int out, int dir);
extern void printmuffetbody(Avl *n, Biobuf *in, int out, int dir);
extern void printmuffetnode(Avl *n, Biobuf *in, int out, int dir);

extern void freemuffetnode(Avl * n);
extern void freemuffetchild(Avl * n);
extern void freemuffetbodyline(Avl * n);
extern void freemuffetlink(Avl * n);
extern void freemuffet_inc_ref(Avl * n);
extern void freemuffet_out_ref(Avl * n);
extern void freemuffettag(Avl * n);


typedef struct muffetnode{
	Avl;
	char *name;
	char *host;
	char *bref;
	muffet_type type; 
	
	Avltree *children;
	Avltree *bodylines; //uses tree in maintree node where lnode->name == BODY -> litree
	Avltree *links;
	Avltree *inc_refs;
	Avltree *out_refs;
	Avltree *tags;
} muffetnode;

typedef struct muffetchild{
	Avl;
	char		*name;
	muffetnode	*node;
} muffetchild;

typedef struct muffetbody{
	Avl;
	int index;
	char *line;
	char ** refs;
	char ** parts;
	int refcount;
} muffetbody;

typedef struct muffetlink{
	Avl;
	int		index;
	char	*name;
	char	*url;
	char	*desc;
} muffetlink;

typedef struct muffet_inc_ref{
	Avl;
	char	  	*name;
	muffetnode	*node;
} muffet_inc_ref;

typedef struct muffet_out_ref{
	Avl;
	int		index;
	char	*name;
	char	*text;
	char	*desc;
	muffet_reftype type;
	
} muffet_out_ref;

typedef struct muffettag{
	Avl;
	char 		*name;
	muffetnode 	*node;
} muffettag;


typedef struct tagcountnode{
	int		count;
	char	*name;
} tagcountnode;






void freemuffetnode(Avl * n);
void freemuffetchild(Avl * n);
void freemuffetbody(Avl * n);
void freemuffetlink(Avl * n);
void freemuffet_inc_ref(Avl * n);
void freemuffet_out_ref(Avl * n);
void freemuffettag(Avl * n);


/* === File: muffet/utils.h === */

typedef struct Avl Avl;
typedef struct Avltree Avltree;
typedef struct Biobuf Biobuf;

typedef enum tmptype{
	t_HTML = 0,
	t_GEM,
	t_LINK
}tmptype;

#define MSZ 4
extern Resub match[MSZ];

extern char * htmlstring(char *s);
extern char* cannonizename(char*);
extern char * decannonizename(char * name);
extern int opentemp(char * name);
extern int checkandreplace(char * name, tmptype t);
extern int opentext(char*);
//extern int openlinks(char*);
extern int bgexists(char * name);
//extern int opengem(char * name);
//extern int openhtml(char * name);
extern int mcuistrcmp(char*, char*);
extern char * copymatch(int i);

extern void walk(Avltree *, Biobuf *, int, void(*action)(Avl *, Biobuf *, int, int), int);

extern int mncmp( Avl * i , Avl * j);
extern int mccmp( Avl * i , Avl * j);
extern int mbcmp( Avl * i , Avl * j);
extern int mlcmp( Avl * i , Avl * j);
extern int mircmp( Avl * i , Avl * j);
extern int morcmp( Avl * i , Avl * j);
extern int mtcmp( Avl * i , Avl * j);