304 lines
No EOL
8.8 KiB
Nim
304 lines
No EOL
8.8 KiB
Nim
import strformat
|
|
import tensorCeral
|
|
import streams
|
|
import arraymancer
|
|
import std/stats
|
|
import math
|
|
import sequtils
|
|
import sugar
|
|
import os
|
|
import system
|
|
import algorithm
|
|
import terminal
|
|
import illwill
|
|
import nimpy
|
|
import tables
|
|
|
|
|
|
|
|
illwillInit(fullscreen = false)
|
|
|
|
type data = ref object of RootObj
|
|
originalColor : int
|
|
inputdata : int
|
|
outputChoices : seq[(int,int)]
|
|
distanceFromCorrct : int
|
|
stats : RunningStat
|
|
|
|
proc isCorrect(a : data) : bool =
|
|
return a.originalColor == a.outputChoices[0][1]
|
|
|
|
type commands = enum
|
|
Forward, ForwardGood, ForwardBad, BackwardGood, BackwardBad, Backward, Break, Graph
|
|
|
|
|
|
proc getInput() : commands =
|
|
##input for the statistic Viewer
|
|
while true:
|
|
var key = getKey()
|
|
case key
|
|
of Key.None: discard
|
|
of Key.Escape: quit()
|
|
of Key.Enter:
|
|
return Forward
|
|
of Key.E:
|
|
echo "e"
|
|
return ForwardBad
|
|
of Key.Q:
|
|
return BackwardBad
|
|
of Key.A:
|
|
return BackwardGood
|
|
of Key.D:
|
|
return ForwardGood
|
|
of Key.Space:
|
|
return Backward
|
|
of Key.X:
|
|
return Break
|
|
of Key.W:
|
|
return Graph
|
|
else:
|
|
continue
|
|
|
|
func datainit(a : int, c : int, stats : RunningStat, output : seq[(int,int)]) : data =
|
|
var correct = 0
|
|
for x in output:
|
|
if x[1] == a:
|
|
break
|
|
correct+=1
|
|
break
|
|
result = data(
|
|
originalColor : a,
|
|
inputdata : c,
|
|
outputChoices : output,
|
|
distanceFromCorrct : correct,
|
|
stats : stats
|
|
)
|
|
proc printData(data : data, cerial : (Tensor[float32], Tensor[float32], Tensor[float32])) =
|
|
##prints Data for the statistics viewer
|
|
eraseScreen()
|
|
echo "#"
|
|
echo &"The ordered Choices {data.outputChoices}"
|
|
echo ""
|
|
echo &"Correct Byte {data.originalColor}"
|
|
echo ""
|
|
echo &"Byte Returned {data.outputChoices[0][1]}"
|
|
echo ""
|
|
echo &"input data {cerial[2][data.inputdata, _]}"
|
|
echo ""
|
|
echo &"correct reference color {cerial[0][data.originalColor, _]}"
|
|
echo ""
|
|
echo &"output's reference color {cerial[0][data.outputChoices[0][1], _]}"
|
|
echo ""
|
|
echo &"idk: {data.stats}"
|
|
echo data.originalColor
|
|
echo data.outputChoices[0][1]
|
|
echo "#"
|
|
|
|
var plt = pyImport("matplotlib.pyplot")
|
|
|
|
|
|
proc getMeanValues(a : seq[float32]) : seq[int]=
|
|
var x1 : RunningStat
|
|
var x2 : RunningStat
|
|
var x3 : RunningStat
|
|
for x in countup(0, a.high, 3):
|
|
x1.push a[x]
|
|
for x in countup(1, a.high, 3):
|
|
x2.push a[x]
|
|
for x in countup(2, a.high, 3):
|
|
x3.push a[x]
|
|
|
|
return [x1.mean, x2.mean, x3.mean, x1.kurtosis, x2.kurtosis, x3.kurtosisS].map(x=>x.int)
|
|
proc getByte(input : seq[float32], reference : seq[seq[int]]) : (char, seq[(int,int)], seq[int]) =
|
|
##Gets the most likely byte from an input file, using a reference
|
|
##currently debug and has more inputs and outputs than it needs to
|
|
var lowestdif = @[10000,-1]
|
|
let mean = getMeanValues(input)
|
|
var tempDebug : seq[(int, int)]
|
|
for x in 0 .. reference.high:
|
|
var totaldif = 0
|
|
totaldif = abs(mean[0]-reference[x][0])^2+abs(mean[1]-reference[x][1])^2+abs(mean[2]-reference[x][2])^2
|
|
|
|
if totaldif < lowestdif[0]:
|
|
|
|
lowestdif[1] = x
|
|
lowestdif[0] = totaldif
|
|
|
|
tempDebug.add((totaldif, x))
|
|
return (char(lowestdif[1]), tempDebug, mean)
|
|
proc correlateBadBytes(a : seq[ptr seq[(int,int)]]) : Table[int, Table[int,int]] =
|
|
for x in a:
|
|
var outputChoice = x[][0][1]
|
|
if not result.hasKey(outputChoice):
|
|
result[outputChoice] = initTable[int,int]()
|
|
if result[outputChoice].hasKey(x[][1][1]):
|
|
result[outputChoice][x[][1][1]] += 1
|
|
else:
|
|
result[outputChoice][x[][1][1]] = 1
|
|
|
|
proc getGraphData(a: Table[int, Table[int,int]]) : (seq[int], seq[int], seq[int]) =
|
|
for key,val in a:
|
|
for innerkey, innerval in val:
|
|
result[0].add(key)
|
|
result[1].add(innerkey)
|
|
result[2].add(innerval)
|
|
|
|
|
|
proc getBestBytes*(inputstream : Stream, output : Stream) : ((Tensor[float32],Tensor[float32],Tensor[float32]), seq[seq[int]]) =
|
|
##Takes a cerial inputstream, and a stream to output to
|
|
##Gets the most likely choice from the given data.
|
|
##Curently very simple, can get sub 3% correctness if lucky
|
|
|
|
stderr.writeLine("enter!")
|
|
let cerial = deSerializeTensors(inputstream)
|
|
var reference : seq[seq[int]]
|
|
for x in 0 .. 255:
|
|
let seqn = cerial[2][x, _].toFlatSeq()
|
|
reference.add getMeanValues(seqn)
|
|
|
|
let max = cerial[2].shape[0]-1
|
|
for x in 256 .. max:
|
|
if cerial[2][x, 0 .. 2].toFlatSeq().foldl((a+b)) < 10:
|
|
break
|
|
var chary = getByte((cerial[2][x, _].toFlatSeq()), reference)
|
|
chary[1].sort((a,b)=>cmp(a[0], b[0]))
|
|
output.write chary[0]
|
|
return (cerial, reference)
|
|
|
|
|
|
when isMainModule:
|
|
stderr.writeLine("enter!")
|
|
var params = commandLineParams()
|
|
echo params
|
|
|
|
var input = newFileStream(params[0])
|
|
|
|
var output : FileStream
|
|
if ["", "-"].contains(params[1]):
|
|
output = newFileStream(stdout)
|
|
else:
|
|
output = newFileStream(params[1], fmReadWrite)
|
|
|
|
var (cerial, reference) = getBestBytes(input, output)
|
|
|
|
if defined(colordebug):
|
|
echo params[2]
|
|
var old = newFileStream(params[2], fmRead)
|
|
var outputDebugData : seq[data]
|
|
let max = cerial[2].shape[0]-1
|
|
|
|
output.setPosition(0)
|
|
for x in 256 .. max:
|
|
if cerial[2][x, 0 .. 2].toFlatSeq().foldl((a+b)) < 10:
|
|
break
|
|
var chary = getByte((cerial[2][x, _].toFlatSeq()), reference)
|
|
let originalByte = byte(old.readChar())
|
|
|
|
chary[1].sort((a,b)=>cmp(a[0], b[0]))
|
|
var stats : RunningStat
|
|
for x in chary[1]:
|
|
stats.push x[0]
|
|
outputDebugData.add datainit(originalByte.int, x, stats, chary[1])
|
|
|
|
old.setPosition(0)
|
|
var dif = 0
|
|
var total = 0
|
|
var incorrect : seq[ptr seq[(int,int)]]
|
|
|
|
while not old.atEnd:
|
|
let oldint = old.readChar().int
|
|
let newint = output.readChar().int
|
|
if newint != oldint:
|
|
dif+=1
|
|
incorrect.add(addr outputDebugData[total].outputChoices)
|
|
total += 1
|
|
|
|
if defined(graph):
|
|
|
|
var dirtyIncorrect : seq[ptr seq[(int,int)]]
|
|
var dirtyTemp : seq[seq[(int,int)]]
|
|
|
|
proc getAccuracyDecay(a : seq[(int,int)]) : float =
|
|
return collect(for y in 0 .. 2: (a[y+1][0] / a[y][0])*100).foldl(a+b)/3.float
|
|
|
|
for x in outputDebugData:
|
|
let output = getAccuracyDecay(x.outputChoices)
|
|
if output < 300:
|
|
dirtyIncorrect.add(addr x.outputChoices)
|
|
if x.outputChoices[0][1] == 202 and x.outputChoices[1][1] == 173:
|
|
dirtyTemp.add(x.outputChoices)
|
|
|
|
var dirtyGraph = getGraphData(correlateBadBytes(dirtyIncorrect))
|
|
let fig = plt.subplots(1,2,figsize=(1,2))[1]
|
|
var pure = correlateBadBytes(incorrect)
|
|
var graph = getGraphData(pure)
|
|
let totalErrors = graph[2].foldl((a+b))
|
|
let newErrors = dirtyGraph[2].foldl((a+b))
|
|
|
|
discard fig[0].set_ylabel("Original Byte")
|
|
discard fig[0].set_xlabel("Mistaken Byte")
|
|
discard fig[0].legend(loc="upper left")
|
|
discard fig[1].set_ylabel("Original Byte")
|
|
discard fig[1].set_xlabel("Mistaken Byte")
|
|
discard fig[1].legend(loc="upper left")
|
|
discard fig[0].set_title(&"Before statistical isolation of errors:\n {totalErrors}")
|
|
discard fig[1].set_title(&"After statistical isolation of errors: \n {newErrors} \n {(newERRORS / totalErrors)* 100}% increase")
|
|
|
|
discard fig[0].scatter(graph[0], graph[1], graph[2])
|
|
discard fig[1].scatter(dirtyGraph[0], dirtyGraph[1], dirtyGraph[2])
|
|
|
|
discard plt.show()
|
|
|
|
let purey = outputDebugData.filter(x=>x.outputChoices[0][1] == 202 and x.outputChoices[1][1] == 173 and x.originalColor != 202).map(x=>x.outputChoices)
|
|
echo purey.high
|
|
echo dirtyTemp.high
|
|
var stat1 : RunningStat
|
|
for x in purey:
|
|
stat1.push getAccuracyDecay(x)
|
|
echo stat1
|
|
stat1.clear()
|
|
|
|
for x in dirtyTemp:
|
|
stat1.push getAccuracyDecay(x)
|
|
dirtyTemp = dirtyTemp.filter(x=>purey.contains(x))
|
|
echo stat1
|
|
stat1.clear()
|
|
|
|
for x in dirtyTemp:
|
|
stat1.push getAccuracyDecay(x)
|
|
echo stat1
|
|
|
|
var print = true
|
|
var pos = 0
|
|
while defined(pager):
|
|
if print:
|
|
printData(outputDebugData[pos], cerial)
|
|
else:
|
|
print = true
|
|
let command = getInput()
|
|
case command:
|
|
of Break: break
|
|
of Forward:
|
|
if outputDebugData.high > pos:
|
|
pos+=1
|
|
else:
|
|
continue
|
|
of Backward:
|
|
if pos > 0:
|
|
pos-=1
|
|
else:
|
|
continue
|
|
of ForwardBad:
|
|
for x in pos+1 .. outputDebugData.high:
|
|
if outputDebugData[x].originalColor != outputDebugData[x].outputChoices[0][1]:
|
|
pos = x
|
|
break
|
|
of Graph:
|
|
var data = outputDebugData[pos].outputChoices.map(x=>x[0])[0 .. 100]
|
|
var x = collect(for x in 0 .. 100: x)
|
|
discard plt.scatter(x,data)
|
|
discard plt.show()
|
|
discard plt.clf()
|
|
|
|
else:
|
|
break |