Caroline/ArlongParkScrapeAndInterface
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added graphs

Browse source
This commit is contained in:
user 2024-11-20 21:42:13 -05:00
parent 804d0973a9
commit af66912e04
4 changed files with 273 additions and 32 deletions
src
frontend
main.html
nim/searching
dispatch.nimlib.nimserver

289
src/frontend/main.html
View file

@ -4,11 +4,13 @@
<title>Counter Example</title>
<link
rel="stylesheet"
[]
href="https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/pure-min.css"
integrity="sha384-X38yfunGUhNzHpBaEBsWLO+A0HDYOQi8ufWDkZ0k9e0eXz/tH3II7uKZ9msv++Ls"
crossorigin="anonymous"
/>
</head>
<body>
<main>
<div id="top">
@ -25,7 +27,10 @@
<h6 id="query-time">query-time: 0ms</h6>
</div>
</div>
<div id="left-side"></div>
<div id="left-side">
<button id="show-chapter">Show occurence per chapter</button>
<button id="show-time">Show occurence over time</button>
</div>
<div id="right-side">
<div class="query-group">
<label for="before">Time Span</label>
@ -44,7 +49,7 @@
<label for="chapter-max">End:</label>
<input type="number" id="chapter-max" />
</div>
<div class="query-group" id="time-selectors-time" hidden="true">
<div class="query-group" id="time-selectors-time" hidden>
<label for="time-min">Start:</label>
<input type="datetime-local" id="time-min" />
<label for="time-max">End:</label>
@ -64,7 +69,7 @@
<option value="Author-On">On</option>
</select>
<form onsubmit="processForm(event)">
<input hidden="true" id="author-input" />
<input hidden id="author-input" />
</form>
</div>
<div class="query-group">
@ -73,10 +78,27 @@
<option value="Off">Off</option>
<option value="Postid-on">On</option>
</select>
<input hidden type="number" id="postid-input" />
<form onsubmit="processForm(event)">
<input hidden type="number" id="postid-input" />
</form>
</div>
<div class="query-group">
<label for="order">Order-by:</label>
<select id="order">
<option value="Oldest">Oldest</option>
<option value="Newest">Newest</option>
</select>
</div>
<div class="query-group">
<label for="limit">Limit number of results (0 for off)</label>
<input id="number of results" min="0" />
</div>
</div>
<div id="main-body">
<div class="chart">
<canvas hidden id="chart-render"></canvas>
</div>
</div>
<div id="main-body"></div>
</main>
</body>
<div hidden id="table-template" class="user-result">
@ -107,6 +129,7 @@
<div class="post-lower"></div>
</div>
</div>
<style>
main {
width: 95vw;
@ -126,10 +149,20 @@
text-align: center;
}
#left-side {
grid-row: 1 / 21;
grid-row: 3 / 21;
grid-column: 1 / 3;
background-color: blue;
text-align: left;
align-items: left;
padding-right: 1vw;
}
#left-side button {
border: solid black 1px;
}
#left-side * {
padding-left: 0px;
margin-bottom: 2vh;
}
#right-side {
grid-row: 3 / 21;
@ -209,9 +242,7 @@
width: 90%;
height: 100%;
}
#left-side * {
max-width: 100%;
}
.query-group {
display: flex;
flex-direction: column;
@ -245,7 +276,172 @@
}
</style>
<script>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns/dist/chartjs-adapter-date-fns.bundle.min.js"></script>
<script id="chart-lib">
let chart = null;
function downSample(list, interval) {
let i = 0;
let counter = interval;
let result = [];
list.forEach((x) => {
if (counter == interval) {
result.push(x);
counter = 0;
} else {
counter++;
}
});
return result;
}
function countChapters(obj) {
result = {};
obj.forEach((x) => {
if (result[x]) {
result[x]++;
} else {
result[x] = 1;
}
});
return result;
}
function timeToPos(obj) {
let result = {};
let count = 1;
let on = true;
let length = obj.length;
if (obj.length > 1000) {
obj = downSample(obj, 40);
}
obj.forEach((x) => {
if (on) {
if (length > 1000) {
result[new Date(x * 1000).toISOString()] = count * 40;
} else {
result[new Date(x * 1000).toISOString()] = count;
}
on = true;
} else {
on = false;
}
count++;
});
return result;
}
function renderBarChart() {
let mb = document.querySelector("#main-body");
mb.scrollTop = 0;
if (chart != null) {
chart.destroy();
}
let incidents = countChapters(
responseObject.subposts.map((x) => x.chapter),
);
const ctx = document.getElementById("chart-render");
ctx.removeAttribute("hidden");
chart = new Chart(ctx, {
type: "bar",
data: {
labels: Object.keys(incidents),
datasets: [
{
label: "Number Of Posts Found",
data: Object.values(incidents),
borderWidth: 1,
},
],
},
options: {
scales: {
y: {
beginAtZero: true,
},
},
},
});
}
function renderLineChart() {
let mb = document.querySelector("#main-body");
mb.scrollTop = 0;
if (chart != null) {
chart.destroy();
}
const ctx = document.getElementById("chart-render");
ctx.removeAttribute("hidden");
let time = timeToPos(
responseObject.subposts.map((x) => x.creationTime),
);
let max = time[time.length - 1];
let min = time[0];
const data = {
datasets: [
{
label: "My Line Chart",
data: time,
borderColor: "rgb(75, 192, 192)",
fill: false,
},
],
};
const options = {
responsive: true,
plugins: {
legend: {
position: "top",
},
tooltip: {
mode: "nearest",
intersect: false,
},
},
scales: {
x: {
min: min,
max: max,
type: "time",
time: {
unit: "day", // unit for displaying the data (e.g., 'day', 'minute', 'hour', etc.)
tooltipFormat: "yyyy-dd-MM", // format for tooltip
displayFormats: {
day: "yyyy-dd-MM", // format for displaying date on the x-axis
},
},
title: {
display: true,
text: "Date",
},
},
y: {
beginAtZero: true,
title: {
display: true,
text: "Value",
},
},
},
};
chart = new Chart(ctx, {
type: "line",
data: data,
options: options,
});
}
const chapterbutton = document.getElementById("show-chapter");
const timebutton = document.getElementById("show-time");
chapterbutton.addEventListener("click", (event) => {
renderBarChart();
});
timebutton.addEventListener("click", (event) => {
renderLineChart();
});
</script>
<script id="lib-layout">
function setUrl(params) {
let url = new URL(window.location.href);
let searchCahce = url.search;
@ -265,6 +461,7 @@
apiLocation.pathname = path;
let queryResponse = await fetch(apiLocation);
let qObject = await queryResponse.json();
responseObject = qObject;
return qObject;
}
@ -333,6 +530,7 @@
urls.EndChapter = ChapterMax.value;
break;
case "Time":
console.log("time");
urls.StartTime = new Date(TimeMin.value).getTime() / 1000;
urls.EndTime = new Date(TimeMax.value).getTime() / 1000;
break;
@ -348,11 +546,13 @@
setUrl(urls);
window.location.reload();
}
function setUserNameUrl() {
let url = { username: SearchInput.value, queryType: "user" };
setUrl(url);
window.location.reload();
}
function setPostsUrl() {
const url = { query: SearchInput.value, queryType: "title" };
@ -361,9 +561,8 @@
}
async function defaultSearch() {
let query = await fetch("http://localhost:5000/query");
let queryBody = await query.json();
pushSubposts(queryBody);
const query = await doApiCall("query", {});
pushSubposts(query);
}
async function processForm(event) {
event.preventDefault();
@ -381,12 +580,15 @@
}
}
}
window.addEventListener("popstate", function () {
window.location.reload();
});
</script>
<script>
<script id="runtime">
let responseObject = {};
let urlParams = {};
function pushSubposts(queryBody) {
console.log(queryBody);
let template = document.querySelector("#template");
@ -394,7 +596,8 @@
let time = document.querySelector("#query-time");
time.textContent = `query-time: ${queryBody.queryTime}ms`;
queryBody.subposts.forEach((x) => {
queryBody.subposts.slice(0, 99).forEach((x) => {
console.log("hello");
let cloned = template.cloneNode(true);
let body = x.nonQuotedText.split("\n");
let lower = cloned.querySelector(".post-lower");
@ -497,22 +700,18 @@
mainBody.appendChild(cloned);
}
function convertTime(date) {
return new Date(date.getTime() - date.getTimezoneOffset() * 60000)
.toISOString()
.slice(0, -1);
}
async function main() {
const stats = await fetch("http://localhost:5000/getInfo");
const data = await stats.json();
const data = await doApiCall("getInfo", {});
console.log(data);
const minDate = new Date(data.firstPost * 1000);
const maxDate = new Date(data.lastPost * 1000);
const minDTL = new Date(
minDate.getTime() - minDate.getTimezoneOffset() * 60000,
)
.toISOString()
.slice(0, -1);
const maxDTL = new Date(
maxDate.getTime() - maxDate.getTimezoneOffset() * 60000,
)
.toISOString()
.slice(0, -1);
const minDTL = convertTime(minDate);
const maxDTL = convertTime(maxDate);
let min = document.querySelector("#time-min");
let max = document.querySelector("#time-max");
@ -541,6 +740,8 @@
//
let url = new URL(window.location.href);
const paramsObject = Object.fromEntries(url.searchParams.entries());
urlParams = paramsObject;
if (paramsObject.Submitter != null) {
Author.value = "Author-On";
AuthorInput.removeAttribute("hidden");
@ -554,6 +755,38 @@
if (paramsObject.Fourm != null) {
FourmSelector.value = paramsObject.Fourm;
}
let timespan = document.querySelector("#before");
if (
paramsObject.StartChapter != null ||
paramsObject.EndChapter != null
) {
if (paramsObject.StartChapter != null) {
chaptermin.value = paramsObject.StartChapter;
}
if (paramsObject.EndChapter != null) {
chaptermax.value = paramsObject.EndChapter;
}
timespan.value = "Chapter";
let selector = document.querySelector("#time-selectors-chapter");
selector.removeAttribute("hidden");
} else if (
paramsObject.StartTime != null ||
paramsObject.EndTime != null
) {
if (paramsObject.StartTime != null) {
min.value = convertTime(
new Date(paramsObject.StartTime * 1000),
);
}
if (paramsObject.EndTime != null) {
max.value = convertTime(new Date(paramsObject.EndTime * 1000));
}
console.log("here");
timespan.value = "Time";
let selector = document.querySelector("#time-selectors-time");
selector.removeAttribute("hidden");
}
if (paramsObject.queryType != null) {
switch (paramsObject.queryType) {

1
src/nim/searching/dispatch.nim
View file

@ -57,6 +57,7 @@ proc initHandler*[T](initFunction : proc() : T{.gcsafe.},
threads.setLen(cpus)
for x in 0 .. cpus-1:
createThread(threads[x], DoWork[T], (result.Scheduler, result.communications[x]))
sleep 10
echo x
proc dispatch*[T](request : Request, handler : ThreadHandler[T]) =

15
src/nim/searching/lib.nim
View file

@ -55,7 +55,8 @@ proc rawOverlaps(user : string, totalSubmissions : CountTable[string]) : Table[s
]#
proc fixJsony(a : var string) =
#JSONy doesn't remove 0x1A from strings, resulting in invalid output
a = a.replace($char(26), """\n""")
let listOfReplaceChars = collect(for x in 0 .. 31: ($char(x), """\n"""))
a = a.multiReplace(listOfReplaceChars)
proc getReplyToStart(db : DbConn, a : var seq[SubPost]) =
var last = a[^1]
@ -172,7 +173,11 @@ proc doQuery*(a : Request, b : ControllerData) =
echo sanitizedString
searchFieldParams.add(&""" And TextSearch.NonQuotedText match """)
searchFieldParams.add("""'"""" & sanitizedString & """"'""")
searchFieldParams.add(" order by CreationTime limit 100")
searchFieldParams.add(" order by CreationTime")
if queryObject.query.isNone():
searchFieldParams.add(" limit 100")
let finalQuery =
if queryObject.query.isSome():
@ -185,6 +190,8 @@ proc doQuery*(a : Request, b : ControllerData) =
echo params
let result = fastRowsTyped[SubPost](b.db, sql(finalQuery), params).toSeq()
output[].subposts = result
output[].queryTime = (getMonoTime()-t1).inMilliseconds()
var jsonResult = toJson output[]
fixJsony jsonResult
@ -202,8 +209,8 @@ proc findPost*(a : Request, b : ControllerData) =
return
let query = "%" & postId.get() & "%"
let found = b.db.fastRows(sql"select Title, PostId from MacroPost where Title like ? limit 100", query).toSeq()
a.respond(404, toJson found, b.headers)
let found = b.db.fastRows(sql"select Title, PostId from MacroPost where Title like ?", query).toSeq()
a.respond(200, toJson found, b.headers)
proc getDbInfo*(a : Request, b : ControllerData) =

BIN
src/nim/searching/server
View file

Binary file not shown.