adam/doordesk-js
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

store content in md files with headers and load into RAM

Browse source
This commit is contained in:
Adam 2023-06-26 20:31:08 -04:00
parent ffdc3a7546
commit c8f3ca5a09
38 changed files with 37 additions and 863 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

132
dennis/src/dennis.py
View file

@ -1,7 +1,11 @@
import os
import markdown
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from datetime import date
from datetime import date, datetime
STATIC_PATH = "./static"
api = FastAPI()
@ -19,121 +23,59 @@ class Article(BaseModel):
date: date
content: str
def get_html(content):
with open('./static/'+content) as file:
return file.read();
def walk_for_md(path: str):
buf = []
fake_db = [
for root, _, files in os.walk(path):
print(files)
mdeez = [f"{root}/{filename}" for filename in files if filename[-2:] == "md"]
Article(
content_type='project',
title='''Lightning''',
date=date(2023, 4, 27),
content=get_html('projects/20230427-lightning.html'),
),
if mdeez:
buf.extend(mdeez)
Article(
content_type='project',
title='''Cartman is public!''',
date=date(2022, 10, 20),
content=get_html('projects/20221020-cartman.html'),
),
return buf
Article(
content_type='game',
title='''fps''',
date=date(2022, 10, 9),
content=get_html('games/fps.html'),
),
def get_articles_from_dir(root_path: str) -> list[Article]:
md = markdown.Markdown(extensions=['meta'])
articles: list[Article] = []
Article(
content_type='game',
title='''balls''',
date=date(2022, 9, 13),
content=get_html('games/balls.html'),
),
for file_path in walk_for_md(root_path):
with open(file_path) as file:
html: str = md.convert(file.read());
meta: dict = md.Meta;
Article(
content_type='game',
title='''adam''', # this stuff will change
date=date(2022, 9, 11),
content=get_html('games/adam.html'),
),
articles.append(
Article(
content_type=meta.get('content_type')[0],
title=meta.get('title')[0],
date=datetime.strptime( meta.get( 'date')[0], '%Y %m %d'),
content=html,
));
Article(
content_type='blog',
title='''It's a post about nothing!''',
date=date(2022, 7, 1),
content=get_html('blog/20220701-progress.html'),
),
return articles;
Article(
content_type='project',
title='''What Goes Into a Successful Reddit Post?''',
date=date(2022, 6, 16),
content=get_html('projects/20220614-reddit.html'),
),
Article(
content_type='blog',
title='''Back to School''',
date=date(2022, 6, 2),
content=get_html('blog/20220602-back.html'),
),
Article(
content_type='game',
title='''snek''',
date=date(2022, 5, 29),
content=get_html('games/snek.html'),
),
Article(
content_type='project',
title='''Predicting Housing Prices''',
date=date(2022, 5, 29),
content=get_html('projects/20220529-housing.html'),
),
Article(
content_type='blog',
title='''It's about time, NVIDIA''',
date=date(2022, 5, 20),
content=get_html('blog/20220520-nvidia.html'),
),
Article(
content_type='blog',
title='''Change''',
date=date(2022, 5, 6),
content=get_html('blog/20220506-change.html'),
),
Article(
content_type='blog',
title='''Hume''',
date=date(2022, 2, 7),
content=get_html('blog/000000000-swim.html'),
),
];
DB = sorted(
get_articles_from_dir(STATIC_PATH),
key=lambda article: article.date,
reverse=True
);
@api.get('/home')
async def serve_home():
return fake_db
return DB
@api.get('/blog')
async def serve_blog():
return [entry for entry in fake_db if entry.content_type == 'blog']
return [entry for entry in DB if entry.content_type == 'blog']
@api.get('/projects')
async def serve_projects():
return [entry for entry in fake_db if entry.content_type == 'project' or entry.content_type == 'game']
return [entry for entry in DB if entry.content_type == 'project' or entry.content_type == 'game']
@api.get('/bots')
async def serve_bots():
return [entry for entry in fake_db if entry.content_type == 'chatbot']
return [entry for entry in DB if entry.content_type == 'chatbot']

158
dennis/static/blog/000000000-swim.html
View file

@ -1,158 +0,0 @@
<article>
<p style="text-align: right;">
April 22, 1958<br />
57 Perry Street<br />
New York City<br />
</p>
Dear Hume,
<p>
You ask advice: ah, what a very human and very dangerous thing to do! For to give advice
to a man who asks what to do with his life implies something very close to egomania. To
presume to point a man to the right and ultimate goal&mdash;to point with a trembling
finger in the RIGHT direction is something only a fool would take upon himself.
</p>
<p>
I am not a fool, but I respect your sincerity in asking my
advice. I ask you though, in listening to what I say, to remember that all advice can
only be a product of the man who gives it. What is truth to one may be a disaster to
another. I do not see life through your eyes, nor you through mine. If I were to attempt
to give you
<i>specific</i> advice, it would be too much like the blind leading the blind.
</p>
<p style="text-align: center;">
<i>
"To be, or not to be: that is the question: Whether 'tis nobler in the mind to
suffer the slings and arrows of outrageous fortune, or to take arms against a sea of
troubles..."
</i>
<br />
(Shakespeare)
</p>
<p>
And indeed, that IS the question: whether to float with the tide, or to swim for a goal.
It is a choice we must all make consciously or unconsciously at one time in our lives.
So few people understand this! Think of any decision you've ever made which had a
bearing on your future: I may be wrong, but I don't see how it could have been anything
but a choice however indirect&mdash;between the two things I've mentioned: the floating
or the swimming.
</p>
<p>
But why not float if you have no goal? That is another question.
It is unquestionably better to enjoy the floating than to swim in uncertainty. So how
does a man find a goal? Not a castle in the stars, but a real and tangible thing. How
can a man be sure he's not after the "big rock candy mountain," the enticing sugar-candy
goal that has little taste and no substance?
</p>
<p>
The answer&mdash;and, in a sense, the tragedy of life&mdash;is
that we seek to understand the goal and not the man. We set up a goal which demands of
us certain things: and we do these things. We adjust to the demands of a concept which
CANNOT be valid. When you were young, let us say that you wanted to be a fireman. I feel
reasonably safe in saying that you no longer want to be a fireman. Why? Because your
perspective has changed. It's not the fireman who has changed, but you. Every man is the
sum total of his reactions to experience. As your experiences differ and multiply, you
become a different man, and hence your perspective changes. This goes on and on. Every
reaction is a learning process; every significant experience alters your perspective.
</p>
<p>
So it would seem foolish, would it not, to adjust our lives to
the demands of a goal we see from a different angle every day? How could we ever hope to
accomplish anything other than galloping neurosis?
</p>
<p>
The answer, then, must not deal with goals at all, or not with
tangible goals, anyway. It would take reams of paper to develop this subject to
fulfillment. God only knows how many books have been written on "the meaning of man" and
that sort of thing, and god only knows how many people have pondered the subject. (I use
the term "god only knows" purely as an expression.) There's very little sense in my
trying to give it up to you in the proverbial nutshell, because I'm the first to admit
my absolute lack of qualifications for reducing the meaning of life to one or two
paragraphs.
</p>
<p>
I'm going to steer clear of the word "existentialism," but you
might keep it in mind as a key of sorts. You might also try something called
<i>Being and Nothingness</i> by Jean-Paul Sartre, and another little thing called
<i>Existentialism: From Dostoyevsky to Sartre.</i> These are merely suggestions. If
you're genuinely statisfied with what you are and what you're doing, then give those
books a wide berth. (Let sleeping dogs lie.) But back to the answer. As I said, to put
our faith in tangible goals would seem to be, at best, unwise. So we do not strive to be
firemen, we do not strive to be bankers, nor policemen, nor doctors. WE STRIVE TO BE
OURSELVES.
</p>
<p>
But don't misunderstand me. I don't mean that we can't BE
firemen, bankers, or doctors&mdash;but that we must make the goal conform to the
individual, rather than make the individual conform to the goal. In every man, heredity
and environment have combined to produce a creature of certain abilities and
desires&mdash;including a deeply ingrained need to function in such a way that his life
will be MEANINGFUL. A man has to BE something; he has to matter.
</p>
<p>
As I see it then, the formula runs something like this: a man
must choose a path which will let his ABILITIES function at maximum efficiency toward
the gratification of his DESIRES. In doing this, he is fulfilling a need (giving himself
identity by functioning in a set pattern toward a set goal) he avoids frustrating his
potential (choosing a path which puts no limit on his self-development), and he avoids
the terror of seeing his goal wilt or lose its charm as he draws closer to it (rather
than bending himself to meet the demands of that which he seeks, he has bent his goal to
conform to his own abilities and desires).
</p>
<p>
In short, he has not dedicated his life to reaching a
pre-defined goal, but he has rather chosen a way of like he KNOWS he will enjoy. The
goal is absolutely secondary: it is the
<i>functioning toward the goal</i> which is important. And it seems almost ridiculous to
say that a man MUST function in a pattern of his own choosing; for to let another man
define your own goals is to give up one of the most meaningful aspects of life&mdash;the
definitive act of will which makes a man an individual.
</p>
<p>
Let's assume that you think you have a choice of eight paths to
follow (all pre-defined paths, of course). And let's assume that you can't see any real
purpose in any of the eight. Then&mdash;and here is the essence of all I've
said&mdash;you MUST FIND A NINTH PATH.
</p>
<p>
Naturally, it isn't as easy as it sounds. you've lived a
relatively narrow life, a vertical rather than a horizontal existence. So it isn't any
too difficult to understand why you seem to feel the way you do. But a man who
procrastinates in his CHOOSING will inevitably have his choice made for him by
circumstance.
</p>
<p>
So if you now number yourself among the disenchanted, then you
have no choice but to accept things as they are, or to seriously seek something else.
But beware of looking for
<i>goals</i>: look for a way of life. Decide how you want to live and then see what you
can do to make a living WITHIN that way of life. But you say, "I don't know where to
look; I don't know what to look for."
</p>
<p>
And there's the crux. Is it worth giving up what I have to look
for something better? I don't know&mdash;is it? Who can make that decision but you? But
even by DECIDING TO LOOK, you go a long way toward making the choice.
</p>
<p>
If I don't call this to a halt, I'm going to find myself writing
a book. I hope it's not as confusing as it looks at first glance. Keep in mind, of
course, that this is MY WAY of looking at things. I happen to think that it's pretty
generally applicable, but you may not. Each of us has to create our own credo&mdash;this
merely happens to be mine.
</p>
<p>
If any part of it doesn't seem to make sense, by all means call
it to my attention. I'm not trying to send you out "on the road" in search of Valhalla,
but merely pointing out that it is not necessary to accept the choices handed down to
you by life as you know it. There is more to it than that&mdash;no one HAS to do
something he doesn't want to do for the rest of his life. But then again, if that's what
you wind up doing, by all means convince yourself that you HAD to do it. You'll have
lots of company.
</p>
<br />
<p>And that's it for now. Until I hear from you again, I remain,</p>
<p style="text-align: right;">
your friend...<br />
Hunter
</p>
</article>

0
dennis/test/blog/000000000-swim.md → dennis/static/blog/000000000-swim.md
View file

52
dennis/static/blog/20220506-change.html
View file

@ -1,52 +0,0 @@
<article>
<p style="text-align: center;">
<i>"Life should not be a journey to the grave with the intention of arriving safely in
a pretty and well preserved body, but rather to skid in broadside in a cloud of
smoke, thoroughly used up, totally worn out, and loudly proclaiming "Wow! What a
Ride!"</i>
<br />
(Hunter S.Thompson)
</p>
<p>
There comes a time in one's life, perhaps multiple, when there
is an unquestionable need for change. Maybe you're not sure how, why, or where it came
from, or where even it is you're headed, or how to get there, but here you are taking
your first steps toward a new life. A journey into the unknown. I've just set out on one
of these journeys, and even as I sit here typing this now I can't help but feel a little
bit nervous, but even more excited. I have absolutely no idea where I'm headed to be
quite honest. But I know where I've been.
</p>
<p>
Growing up I would always be taking things apart, I HAD to see
what was inside. What makes this thing, a thing. What makes it tick? Can it tick faster?
For no particular reason I just had to know every little detail about what made the
thing the thing that it was and why it did what it did. It's a gift and a curse of
sorts. Quickly this led to taking apart things of increasing complexity, our home
computer for instance. Luckily I was able to get it put back together before my parents
got home because it was made clear that this was not allowed, and the CPU didn't seem to
mind the sudden absence of thermal compound either. I must have been around 7 or 8 years
old at that time, and it still puzzles me just what is going on inside there.
</p>
<p>
I have a better idea now, naturally I had to figure out just
what all those pieces were, what they did, and how they did it. What if I replaced some
of these parts with other parts? As I honed my web searching skills to try to answer the
seemingly endless hows and whys I ended up building myself a little hotrod computer and
then raced it against other peoples' computers because why not, right? And I actually
won! It was an overclocking contest called the winter suicides, a kind of computer drag
race. Highest CPU clock speed wins, you have to boot into Windows XP, open CPU-Z, and
take a screenshot. If it crashes immediately after that (and it did) it still counts. I
got some pretty weird looks from my father as I stuck my computer outside in the snow
but that was a small price to pay for the grand prize which was a RAM kit (2GB of DDR400
I believe) and RAM cooler.
</p>
<p>
After getting comfortable with hardware I started to study the
software side of things, I tried teaching myself C++ (and didn't get very far), I did
teach myself HTML and CSS, some JavaScript, and started playing around with Linux. It
took until only a year or two ago to finally be completely on Linux full time (gaming
holding me back), I even have a Linux phone now (Pinephone Pro). At this point I reached
high school and my attention moved from computers to cars.
</p>
<p>To be continued...</p>
</article>

0
dennis/test/blog/20220506-change.md → dennis/static/blog/20220506-change.md
View file

11
dennis/static/blog/20220520-nvidia.html
View file

@ -1,11 +0,0 @@
<article>
<p>It's about time... NVIDIA has finally released and is starting to
support Open-source software with their new modules released recently for the Linux
kernel. NVIDIA historically has been seemingly against Linux/OSS for whatever reason.
This is a huge step forward both for end users and NVIDIA.
</p>
<p style="text-align: center;">
<a href="https://github.com/NVIDIA/open-gpu-kernel-modules">
NVIDIA open-gpu-kernel-modules</a > on github.
</p>
</article>

0
dennis/test/blog/20220520-nvidia.md → dennis/static/blog/20220520-nvidia.md
View file

43
dennis/static/blog/20220602-back.html
View file

@ -1,43 +0,0 @@
<article>
<h3>Where the hell have I been!?</h3>
<p>
Looking back at the past 5 weeks, it's impressive the amount of new things that have
been shoved in my face. A list I'll try to make contains:
</p>
<ul>
<li><a href="https://www.python.org/">Python</a></li>
<li><a href="https://pandas.pydata.org/">Pandas</a></li>
<li><a href="https://matplotlib.org/">Matplotlib</a></li>
<li><a href="https://seaborn.pydata.org/">Seaborn</a></li>
<li><a href="https://www.statsmodels.org/">Statsmodels</a></li>
<li><a href="https://scikit-learn.org/">Scikit-Learn</a></li>
<li>
<a href="https://www.crummy.com/software/BeautifulSoup/">Beautiful Soup</a>
</li>
<li><a href="https://www.selenium.dev/">Selenium</a></li>
<li><a href="https://github.com/praw-dev/praw">PRAW</a></li>
<li>Plus the math and background to go with it all!</li>
</ul>
<p>
It doesn't seem like much at the time except chaos, but then about a week later it
finally sets in. After tomorrow we'll be halfway through the course and while I guess
you could say that it's half over, or that it signifies progress, I feel it's more like
being halfway up Mount Everest and looking&mdash;trying to squint through the clouds and
make out what looks like the peak. I don't see a peak and maybe it's because I'm
nearsighted but I can also tell you that if were to look down then I can't see where
I've started either!
</p>
<p>
It's been quite a ride and I hope to see it to the end. I don't have time to even think
about it further. It's where I perform my best though, on my heels. Probably by
design...
</p>
<h3>After?</h3>
<p>
I would like to use these skills to expand on some of the class projects I've worked on
and I have some other ideas using language processing I think would be fun to play with.
I think it would be fun to create an internet chat bot, we'll start with text but if
speech recognition is practical then I may add and play with that too. I would also like
to make some sort of "Propaganda Detector"
</p>
</article>

0
dennis/test/blog/20220602-back.md → dennis/static/blog/20220602-back.md
View file

94
dennis/static/blog/20220701-progress.html
View file

@ -1,94 +0,0 @@
<article>
The progress update
<p style='text-align: center;'>
<img src="https://old.doordesk.net/pics/plates.gif" />
</p>
<h3>Bots</h3>
<p>
After finding a number of ways not to begin the project formerly known as my capstone,
I've finally settled on a
<a href="https://www.kaggle.com/datasets/bwandowando/ukraine-russian-crisis-twitter-dataset-1-2-m-rows">dataset</a>.
The project is about detecting bots, starting with twitter. I've
<a href="https://old.doordesk.net/projects/bots/docs/debot.pdf">studied</a> a
<a href="https://old.doordesk.net/projects/bots/docs/botwalk.pdf">few</a>
<a href="https://old.doordesk.net/projects/bots/docs/smu.pdf">different</a>
<a href="https://old.doordesk.net/projects/bots/docs/div.pdf">methods</a> of bot detection and particularly like the
<a href="https://old.doordesk.net/projects/bots/docs/debot.pdf">DeBot</a> and
<a href="https://old.doordesk.net/projects/bots/docs/botwalk.pdf">BotWalk</a> methods and think I will try to mimic them,
in that order.
</p>
<p>
Long story short, DeBot uses a fancy method of time correlation to group accounts
together based on their posting habits. By identifying accounts that all have identical
posting habits that are beyond what a human could do by coincidence, this is a great
first step to identifying an inital group of seed bots. This can then be expanded by
using BotWalk's method of checking all the followers of the bot accounts and comparing
anomalous behavior to separate humans from non-humans. Rinse and repeat. I'll begin this
on twitter but hope to make it platform independent.
</p>
<h3>The Real Capstone</h3>
<p>
The bot project is too much to complete in this short amount of time, so instead I'm
working with a
<a href="https://archive-beta.ics.uci.edu/ml/datasets/auto+mpg">small dataset</a>
containing info about cars with some specs and I'll predict MPG. The problem itself for
me is trivial from past study/experience as an auto mechanic so I should have a nice
playground to focus completely on modeling. It's a very small data set too at < 400
lines, I should be able to test multiple models in depth very quickly. It may or may not
be interesting, expect a write-up anyway.
</p>
<h3>Cartman</h3>
<p>
Well I guess I've adopted an 8 year old. Based on
<a href="https://github.com/RuolinZheng08/twewy-discord-chatbot">this project</a>
I've trained a chat bot with the personality of Eric Cartman. He's a feature of my
Discord bot living on a Raspberry Pi 4B, which I would say is probably the slowest
computer you would ever want to run something like this on. It takes a somewhat
reasonable amount of time to respond, almost feeling human if you make it think a bit.
The project uses <a href="https://pytorch.org/">PyTorch</a> to train the model. I'd like
to re-create it using <a href="https://www.tensorflow.org/">TensorFlow</a> as an
exercise to understand each one better, but that's a project for another night. It also
only responds to one line at a time so it can't carry a conversation with context,
yet...
</p>
<h3>Website</h3>
<p>
I never thought I'd end up having a blog. I had no plans at all actually when I set up
this server, just to host a silly page that I would change from time to time whenever I
was bored. I've been looking at
<a href="https://gohugo.io/">Hugo</a> as a way to organize what is now just a list of
divs in a single html file slowly growing out of control. Basically you just dump each
post into its own file, create a template of how to render them, and let it do its
thing. I should be able to create a template that recreates exactly what you see right
now, which is beginning to grow on me.
</p>
<p>
If you haven't noticed yet, (and I don't blame you if you haven't because only a handful
of people even visit this page) each time there is an update there is a completely new
background image, color scheme, a whole new theme. This is because this page is a near
identical representation of terminal windows open my computer and each time I update the
page I also update it with my current wallpaper, which generates the color scheme
dynamically using
<a href="https://github.com/dylanaraps/pywal">Pywal</a>.
</p>
TODO:
<ul>
<li>Code blocks with syntax highlighting</li>
<li>
Develop an easy workflow to dump a jupyter notebook into the website and have it
display nicely with minimal effort
</li>
<li>
Find a way to hack plots generated with matplotlib to change colors with the page
color scheme (or find another way to do the same thing)
</li>
<li>
Automate generating the site - probably
<a href="https://gohugo.io/">Hugo</a>
</li>
<li>Separate from blog, projects, etc.</li>
<li>Add socials, contact, about</li>
<li>A bunch of stuff I haven't even thought of yet</li>
</ul>
<p>That's all for now</p>
</article>

0
dennis/test/blog/20220701-progress.md → dennis/static/blog/20220701-progress.md
View file

6
dennis/static/games/adam.html
View file

@ -1,6 +0,0 @@
<article>
<p>
<a href="https://old.doordesk.net/games/adam/" target="_blank">adam</a> is a quick fps demo to test how well WebGL
performs using <a href="https://unity.com">Unity</a>.
</p>
</article>

0
dennis/test/games/adam.md → dennis/static/games/adam.md
View file

6
dennis/static/games/balls.html
View file

@ -1,6 +0,0 @@
<article>
<p>
<a href="https://old.doordesk.net/games/balls/" target="_blank">balls</a> is another demo to test WebGL performance.
This time using <a href="https://godotengine.org/">Godot Engine</a>.
</p>
</article>

0
dennis/test/games/balls.md → dennis/static/games/balls.md
View file

6
dennis/static/games/fps.html
View file

@ -1,6 +0,0 @@
<article>
<p>
<a href="https://old.doordesk.net/games/fps/" target="_blank">fps</a> is a Godot/WebGL experiment from scratch with
multiplayer using websockets and a master/slave architecture. Invite a friend or open multiple instances!
</p>
</article>

0
dennis/test/games/fps.md → dennis/static/games/fps.md
View file

5
dennis/static/games/snek.html
View file

@ -1,5 +0,0 @@
<article>
<p>
<a href="https://old.doordesk.net/snek" target="_blank">snek</a> is a simple snake game made with JS/Canvas.
</p>
</article>

0
dennis/test/games/snek.md → dennis/static/games/snek.md
View file

136
dennis/static/projects/20220529-housing.html
View file

@ -1,136 +0,0 @@
<article>
<p>
A recent project I had for class was to use
<a href="https://scikit-learn.org/stable/index.html" target="new">scikit-learn</a>
to create a regression model that will predict the price of a house based on some
features of that house.
</p>
<h3>How?</h3>
<ol>
<li>
Pick out and analyze certain features from the dataset. Used here is the
<a href="https://www.kaggle.com/datasets/marcopale/housing" target="new"
>Ames Iowa Housing Data</a
>
set.
</li>
<li>
Do some signal processing to provide a clearer input down the line, improving
accuracy
</li>
<li>Make predictions on sale price</li>
<li>
Compare the predicted prices to recorded actual sale prices and score the results
</li>
</ol>
<h3>What's important?</h3>
<p>
Well, I don't know much about appraising houses. But I have heard the term "price per
square foot" so we'll start with that:
</p>
<p style="text-align: center;"><img src="https://old.doordesk.net/pics/livarea_no_outliers.png" /></p>
<p>
There is a feature for 'Above Grade Living Area' meaning floor area that's not basement.
It looks linear, there were a couple outliers to take care of but this should be a good
signal.
</p>
<p>Next I calculated the age of every house at time of sale and plotted it:</p>
<p style="text-align: center;"><img src="https://old.doordesk.net/pics/age.png" /></p>
<p>
Exactly what I'd expect to see. Price drops as age goes up, a few outliers. We'll
include that in the model.
</p>
<p>Next I chose the area of the lot:</p>
<p style="text-align: center;"><img src="https://old.doordesk.net/pics/lot_area.png" /></p>
<p>
Lot area positively affects sale price because land has value. Most of the houses here
have similarly sized lots.
</p>
<h3>Pre-Processing</h3>
<div>
<p>
Here is an example where using
<a
href="https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html"
target="new"
>StandardScaler()</a
>
just doesn't cut it. The values are all scaled in a way where they can be compared
to one-another, but outliers have a huge effect on the clarity of the signal as a
whole.
</p>
<span>
<center>
<img src="https://old.doordesk.net/pics/age_liv_area_ss.png" />
<img src="https://old.doordesk.net/pics/age_liv_qt.png" />
</center>
</span>
</div>
<p>
You should clearly see in the second figure that an old shed represented in the top left
corner will sell for far less than a brand new mansion represented in the bottom right
corner. This is the result of using the
<a
href="https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.QuantileTransformer.html"
target="new"
>QuantileTransformer()</a
>
for scaling.
</p>
<h3>The Model</h3>
<p>
A simple
<a
href="https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html"
>LinearRegression()</a
>
should do just fine, with
<a
href="https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.QuantileTransformer.html"
target="new"
>QuantileTransformer()</a
>
scaling of course.
</p>
<center>
<img src="https://old.doordesk.net/pics/mod_out.png" />
</center>
<p>
Predictions were within about $35-$40k on average.<br />
It's a little fuzzy in the higher end of prices, I believe due to the small sample size.
There are a few outliers that can probably be reduced with some deeper cleaning however
I was worried about going too far and creating a different story. An "ideal" model in
this case would look like a straight line.
</p>
<h3>Conclusion</h3>
<p>
This model was designed with a focus on quality and consistency. With some refinement,
the margin of error should be able to be reduced to a reasonable number and then
reliable, accurate predictions can be made for any application where there is a need to
assess the value of a property.
</p>
<p>
I think a large limiting factor here is the size of the dataset compared to the quality
of the features provided. There are
<a href="http://jse.amstat.org/v19n3/decock/DataDocumentation.txt">more features</a>
from this dataset that can be included but I think the largest gains will be had from
simply feeding in more data. As you stray from the "low hanging fruit" features, the
quality of your model overall starts to go down.
</p>
<p>Here's an interesting case, Overall Condition of Property:<br /><br /></p>
<center>
<img src="https://old.doordesk.net/pics/overall_cond.png" />
</center>
<p>
You would expect sale price to increase with quality, no? Yet it goes down.. Why?<br />
I believe it's because a lot of sellers want to say that their house is of highest
quality, no matter the condition. It seems that most normal people (who aren't liars)
dont't care to rate their property and just say it's average. Both of these combined
actually create a negative trend for quality which definitely won't help predictions!
</p>
<p>
I would like to expand this in the future, maybe scraping websites like Zillow to gather
more data. <br />
We'll see.
</p>
</article>

0
dennis/test/projects/20220529-housing.md → dennis/static/projects/20220529-housing.md
View file

BIN
dennis/static/projects/20220529-housing/pics/age.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 80 KiB

BIN
dennis/static/projects/20220529-housing/pics/age_liv_area_ss.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 140 KiB

BIN
dennis/static/projects/20220529-housing/pics/age_liv_qt.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 252 KiB

BIN
dennis/static/projects/20220529-housing/pics/livarea_no_outliers.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 79 KiB

BIN
dennis/static/projects/20220529-housing/pics/lot_area.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 65 KiB

BIN
dennis/static/projects/20220529-housing/pics/mod_out.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 128 KiB

BIN
dennis/static/projects/20220529-housing/pics/overall_cond.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 48 KiB

126
dennis/static/projects/20220614-reddit.html
View file

@ -1,126 +0,0 @@
<article>
<p>
In an attempt to find out what about a Reddit post makes it successful I will use some
classification models to try to determine which features have the highest influence on
making a correct prediction. In particular I use
<a
href="https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html"
>Random Forest</a
>
and
<a
href="https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html"
>KNNeighbors</a
>
classifiers. Then I'll score the results and see what the highest predictors are.
</p>
<p>
To find what goes into making a successful Reddit post we'll have to do a few things,
first of which is collecting data:
</p>
<h3>Introducing Scrapey!</h3>
<p>
<a href="https://old.doordesk.net/projects/reddit/scrapey.html">Scrapey</a> is my scraper script that takes a snapshot
of Reddit/r/all hot and saves the data to a .csv file including a calculated age for
each post about every 12 minutes. Run time is about 2 minutes per iteration and each
time adds about 100 unique posts to the list while updating any post it's already seen.
</p>
<p>
I run this in the background in a terminal and it updates my data set every ~12 minutes.
I have records of all posts within about 12 minutes of them disappearing from /r/all.
</p>
<h3>EDA</h3>
<p>
<a href="https://old.doordesk.net/projects/reddit/EDA.html">Next I take a quick look to see what looks useful</a>, what
doesn't, and check for outliers that will throw off the model. There were a few outliers
to drop from the num_comments column.
</p>
Chosen Features:
<ul>
<li>Title</li>
<li>Subreddit</li>
<li>Over_18</li>
<li>Is_Original_Content</li>
<li>Is_Self</li>
<li>Spoiler</li>
<li>Locked</li>
<li>Stickied</li>
<li>Num_Comments (Target)</li>
</ul>
<p>
Then I split the data I'm going to use into two dataframes (numeric and non) to prepare
for further processing.
</p>
<h3>Clean</h3>
<p><a href="https://old.doordesk.net/projects/reddit/clean.html">Cleaning the data further</a> consists of:</p>
<ul>
<li>Scaling numeric features between 0-1</li>
<li>Converting '_' and '-' to whitespace</li>
<li>Removing any non a-z or A-Z or whitespace</li>
<li>Stripping any leftover whitespace</li>
<li>Deleting any titles that were reduced to empty strings</li>
</ul>
<h3>Model</h3>
<p>
If the number of comments of a post is greater than the median total number of comments
then it's assigned a 1, otherwise a 0. This is the target column. I then try some
lemmatizing, it doesn't seem to add much. After that I create and join some dummies,
then split and feed the new dataframe into
<a
href="https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html"
>Random Forest</a
>
and
<a
href="https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html"
>KNNeighbors</a
>
classifiers. Both actually scored the same with
<a
href="https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.cross_validate.html"
>cross validation</a
>
so I mainly used the forest.
</p>
<p><a href="https://old.doordesk.net/projects/reddit/model.html">Notebook Here</a></p>
<h3>Conclusion</h3>
<p>Some Predictors from Top 25:</p>
<ul>
<li>Is_Self</li>
<li>Subreddit_Memes</li>
<li>OC</li>
<li>Over_18</li>
<li>Subreddit_Shitposting</li>
<li>Is_Original_Content</li>
<li>Subreddit_Superstonk</li>
</ul>
<p>
Popular words: 'like', 'just', 'time', 'new', 'oc', 'good', 'got', 'day', 'today', 'im',
'dont', and 'love'.
</p>
<p>
People on Reddit (at least in the past few days) like their memes, porn, and talking
about their day. And it's preferred if the content is original and self posted. So yes,
post your memes to memes and shitposting, tag them NSFW, use some words from the list,
and rake in all that sweet karma!
</p>
<p>
But it's not that simple, this is a fairly simple model, with simple data. To go beyond
this I think the comments would have to be analyzed.
<a href="https://en.wikipedia.org/wiki/Lemmatisation">Lemmatisation</a> I thought would
be the most influential piece, and I still think that thinking is correct. But in this
case it doesn't apply because there is no real meaning to be had from reddit post
titles, at least to a computer. (or I did something wrong)
</p>
<p>
There's a lot more seen by a human than just the text in the title, there's often an
image attached, most posts reference a recent/current event, they could be an inside
joke of sorts. For some posts there could be emojis in the title, and depending on their
combination they can take on a meaning completely different from their individual
meanings. The next step from here I believe is to analyze the comments section of these
posts because in this moment I think that's the easiest way to truly describe the
meaning of a post to a computer. With what was gathered here I'm only to get 10% above
baseline and I think that's all there is to be had here, I mean we can tweak for a few
percent probably but I don't think there's much left on the table.
</p>
</article>

0
dennis/test/projects/20220614-reddit.md → dennis/static/projects/20220614-reddit.md
View file

33
dennis/static/projects/20221020-cartman.html
View file

@ -1,33 +0,0 @@
<article>
<p>
<a href="https://old.doordesk.net/cartman">Cartman</a> is trained by combining Microsoft's
<a href="https://huggingface.co/microsoft/DialoGPT-large">DialoGPT-large</a>
NLP model (trained on 147M samples of multi-turn dialogue from Reddit) with 17 seasons of
<a href="https://southparkstudios.com">South Park</a>
transcripts.
</p>
<p>
Requests are routed from
<a href="https://nginx.com">Nginx</a>
through
<a href="https://www.wireguard.com">WireGuard</a>
to a
<a href="https://www.tomshardware.com/news/raspberry-pi-4-8gb-tested">Raspberry Pi 4B 8GB</a>
running
<a href="https://fastapi.tiangolo.com">FastAPI</a>,
and the Cartman model using <a href="https://pytorch.org">PyTorch</a>.
It has enough RAM for more, but the CPU is pretty much at its limit. Expect it to take a few
seconds, I'm cheap. Sorry(kinda).
</p>
<p>
You can download a Docker image if you'd like to run it on your own hardware for either
<a href="https://old.doordesk.net/files/chatbots_api_x86_64.tar.gz">x86_64</a>
or
<a href="https://old.doordesk.net/files/chatbots_api_aarch64.tar.gz">aarch64</a>.
</p>
<p>
More info <a href="https://github.com/adoyle0/cartman">here</a> as well as
<a href="https://github.com/adoyle0/cartman/tree/master/api/test">example scripts</a>
to talk to the docker container.
</p>
</article>

0
dennis/test/projects/20221020-cartman.md → dennis/static/projects/20221020-cartman.md
View file

11
dennis/static/projects/20230427-lightning.html
View file

@ -1,11 +0,0 @@
<article>
<p>
<a href="https://lightning.doordesk.net" target="_blank">Lightning</a> is a mapping/data vis project for finding
EV charging stations. It uses <a href="https://github.com/maplibre/martin" target="_blank">Martin</a> to serve
tiles generated from <a href="https://www.openstreetmap.org" target="_blank">OpenStreetMap</a> data to a <a
href="https://maplibre.org/" target="_blank">MapLibre</a> frontend. Additional layers are added on top
via <a href="https://deck.gl" target="_blank">Deck.gl</a> using data from <a
href="https://github.com/kevin-fwu/EVChargerFinder" target="_blank">EVChargerFinder</a> made by my friend
Kevin.
</p>
</article>

0
dennis/test/projects/20230427-lightning.md → dennis/static/projects/20230427-lightning.md
View file

1
dennis/test/bots/cartman.html
View file

@ -1 +0,0 @@
cartman

53
dennis/test/filetest.py
View file

@ -1,53 +0,0 @@
import os
import markdown
from datetime import date, datetime
from pydantic import BaseModel
class Article(BaseModel):
content_type: str
title: str
date: date
content: str
def walk_for_md(path: str):
buf = []
for root, _, files in os.walk(path):
mdeez = [f"{root}/{filename}" for filename in files if filename[-2:] == "md"]
if mdeez:
buf.extend(mdeez)
return buf
def get_articles_from_dir(root_path: str) -> list[Article]:
md = markdown.Markdown(extensions=['meta'])
articles: list[Article] = []
for file_path in walk_for_md(root_path):
with open(file_path) as file:
html: str = md.convert(file.read());
meta: dict = md.Meta;
articles.append(
Article(
content_type=meta.get('content_type')[0],
title=meta.get('title')[0],
date=datetime.strptime( meta.get( 'date')[0], '%Y %m %d'),
content=html,
));
return articles;
DB = sorted(
get_articles_from_dir('.'),
key=lambda article: article.date,
reverse=True
);
for article in DB:
print(
article.date,
article.content_type,
article.title,
)

23
dennis/test/games/index.html
View file

@ -1,23 +0,0 @@
<h3>Some games using wasm/webgl</h3>
<p>Browser performance as of January 2023</p>
<p>Tested better:</p>
<ol>
<li>Opera</li>
<li>Firefox Developer Edition</li>
<li>Brave</li>
</ol>
<p>Tested poor or broken:</p>
<ol>
<li>Safari</li>
<li>Chrome stable release or older</li>
<li>Edge, see above^</li>
</ol>
<p>Consider anything else average or let me know otherwise</p>
<ul>
---MY GAMES---
<li><a href="https://old.doordesk.net/games/adam">adam</a> - The first. Unity Demo/Tutorial with some mods</li>
<li><a href="https://old.doordesk.net/games/fps">multiplayer fps</a> - Dive into netcode with Godot (Open two, invite
your friends!)</li>
<li><a href="https://old.doordesk.net/games/../snek">snek</a> - Canvas + JS (the actual first)</li>
<li><a href="https://old.doordesk.net/games/balls">balls</a> - Godot demo engine test</li>
</ul>

4
dennis/test/runtest
View file

@ -1,4 +0,0 @@
#!/bin/bash
clear &&
python filetest.py