Page MenuHomeSealhub

No OneTemporary

diff --git a/src/index.ts b/src/index.ts
index 0130d70..40fded7 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,523 +1,519 @@
import { list as suffix_list } from "./suffix_list.js";
import * as Base64 from "base64-js";
import * as Hex from "@smithy/util-hex-encoding";
import pako from "pako";
import * as HAR from "har-format";
export type ValPath = HARValRoute[];
interface HARValRoute {
pretty_print(): string;
}
export function val_path_cmp(v1: ValPath, v2: ValPath): boolean {
if (v1.length != v2.length) return false;
for (let i = 0; i < v1.length; i++) {
if (v1[i].pretty_print() !== v2[i].pretty_print()) return false;
}
return true;
}
export interface HARRecord {
id: number;
hostname: string;
path: ValPath;
value: string;
private_info_idxs: number[];
req_id: number;
importance: number;
}
export interface SummaryRecord {
private_info_idxs: Set<number>;
importance: number;
num_entries: number;
num_entries_with_priv_info: number;
}
interface HARHeader {
name: string;
value: string;
}
export class HeaderRoute implements HARValRoute {
type: "header";
name: string;
constructor(name: string) {
this.name = name;
}
pretty_print(): string {
return "Header('" + this.name + "')";
}
}
export class BodyRoute implements HARValRoute {
type: "body";
constructor() {}
pretty_print(): string {
return "Body";
}
}
export class URLParamRoute implements HARValRoute {
type: "urlparam";
name: string;
constructor(name: string) {
this.name = name;
}
pretty_print(): string {
return "UrlParam('" + this.name + "')";
}
}
export class SplitByRoute implements HARValRoute {
type: "splitby";
by: string;
constructor(by: string) {
this.by = by;
}
pretty_print(): string {
return "SplitBy('" + this.by + "')";
}
}
export class JSONDecodeRoute implements HARValRoute {
type: "jsondecode";
constructor() {}
pretty_print(): string {
return "JSONDecode()";
}
}
export class HexEncodeRoute implements HARValRoute {
type: "hexencode";
constructor() {}
pretty_print(): string {
return "HexEncode()";
}
}
export class IdxRoute implements HARValRoute {
type: "idx";
idx: number;
constructor(idx: number) {
this.idx = idx;
}
pretty_print(): string {
return "Idx('" + this.idx + "')";
}
}
export class KeyRoute implements HARValRoute {
type: "key";
key: string;
constructor(key: string) {
this.key = key;
}
pretty_print(): string {
return "Key('" + this.key + "')";
}
}
export class Base64DecodeRoute implements HARValRoute {
type: "unbase64";
constructor() {}
pretty_print(): string {
return "Base64Decode()";
}
}
export class UnbinaryString implements HARValRoute {
type: "unbinary_str";
constructor() {}
pretty_print(): string {
return "BinaryToString()";
}
}
export class InflateRoute implements HARValRoute {
type: "inflate";
constructor() {}
pretty_print(): string {
return "Inflate()";
}
}
export class HARParser {
har_entries: HAR.Entry[] | undefined;
entries: HARRecord[] = [];
summary: Map<String, SummaryRecord> = new Map();
suffixes: Set<String>;
// data -> desc
private_information: [string, string][];
curr_id: number = 0;
constructor() {
this.suffixes = new Set(suffix_list);
}
get_real_domain(host: string): string {
let subdomains = host.split(".");
let i = 1;
let curr_string;
for (; i <= subdomains.length; i++) {
curr_string = "";
let host_sections = subdomains.slice(subdomains.length - i);
for (const [idx, s] of host_sections.entries()) {
curr_string += s;
if (idx != host_sections.length - 1) curr_string += ".";
}
if (!this.suffixes.has(curr_string)) break;
}
if (curr_string === undefined)
throw new Error("failed to get a domain from: " + host);
return curr_string;
}
parseHeaders(
headers: HARHeader[],
curr_path: ValPath,
hostname: string,
req_id: number
) {
for (const { name, value } of headers) {
let new_path = [...curr_path, new HeaderRoute(name)];
this.parseEntry(value, new_path, hostname, req_id);
}
}
parseURLParams(
params: URLSearchParams,
curr_path: ValPath,
hostname: string,
req_id: number
) {
let i = 0;
for (const [key, value] of params.entries()) {
let new_path: ValPath = [...curr_path, new URLParamRoute(key)];
this.parseEntry(value, new_path, hostname, req_id);
new_path = [...curr_path, new IdxRoute(i)];
this.parseEntry(key, new_path, hostname, req_id);
i++;
}
}
tryParseUrlencodedParams(
s: string,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
let decoded_everything = true;
s.split("&")
.filter((e) => {
if (e.indexOf("=") !== -1) return true;
decoded_everything = false;
return false;
})
.entries()
.forEach(([idx, e]) => {
let i = e.indexOf("=");
let k;
let v;
try {
v = decodeURIComponent(e.slice(i + 1).replace(/\+/g, " "));
k = decodeURIComponent(e.slice(0, i).replace(/\+/g, " "));
} catch (_) {
decoded_everything = false;
}
if (v && k) {
let new_path = [...curr_path, new URLParamRoute(k)];
this.parseEntry(v, new_path, hostname, req_id);
new_path = [...curr_path, new IdxRoute(idx)];
this.parseEntry(k, new_path, hostname, req_id);
}
});
return decoded_everything;
}
tryParseDeflate(
s: Uint8Array,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
try {
let result = pako.inflate(s);
this.parseBinaryEntry(
result,
[...curr_path, new InflateRoute()],
hostname,
req_id
);
} catch (e) {
return false;
}
return true;
}
tryParseBinaryToString(
entry: Uint8Array,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
try {
let decoded_str = new TextDecoder("utf8", {
fatal: true,
}).decode(entry);
this.parseEntry(
decoded_str,
[...curr_path, new UnbinaryString()],
hostname,
req_id
);
} catch (_) {
return false;
}
return true;
}
tryParseBase64(
s: string,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
if (s.length == 0) return false;
let decoded_str: string = "";
let firt_non_base64 =
/([^ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+=\/])/;
let contains_at_least_2 =
/([^ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+=\/]).*([^ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+=\/])/;
if (contains_at_least_2.test(s)) return false;
let matches = firt_non_base64.exec(s);
let matches_uniq = new Set(matches);
let is_base64 = matches_uniq.size === 0;
if (is_base64) {
try {
// works for the browser
let buff = Base64.toByteArray(s);
this.parseBinaryEntry(
buff,
[...curr_path, new Base64DecodeRoute()],
hostname,
req_id
);
} catch (_) {
is_base64 = false;
}
} else {
if (!matches_uniq) return false;
if (matches_uniq.size != 1) return false;
let everything_decoded = true;
let split_char: string = matches_uniq.values().next().value;
let substrs = s.split(split_char);
curr_path.push(new SplitByRoute(split_char));
let idx = 0;
for (const substr of substrs) {
if (substr.length == 0) continue;
let new_path = [...curr_path, new IdxRoute(idx)];
everything_decoded =
this.tryParseBase64(substr, new_path, hostname, req_id) &&
everything_decoded;
idx++;
}
return everything_decoded;
}
if (decoded_str.length == 0) return false;
curr_path.push(new Base64DecodeRoute());
this.parseEntry(decoded_str, curr_path, hostname, req_id);
return true;
}
loopThroughJSON(
obj: any,
curr_path: ValPath,
hostname: string,
req_id: number
) {
let i = 0;
for (let key in obj) {
let path_idx = [...curr_path, new IdxRoute(i)];
this.parseEntry(key, path_idx, hostname, req_id);
let path_key = [...curr_path, new KeyRoute(key)];
if (typeof obj[key] === "object") {
if (Array.isArray(obj[key])) {
for (let i = 0; i < obj[key].length; i++) {
let new_path = [...path_key, new IdxRoute(i)];
this.loopThroughJSON(
obj[key][i],
new_path,
hostname,
req_id
);
}
} else {
this.loopThroughJSON(obj[key], path_key, hostname, req_id);
}
} else {
this.parseEntry(obj[key] + "", path_key, hostname, req_id);
}
}
i++;
}
tryParseJSON(
s: string,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
if (s.length == 0) return false;
let json_obj;
try {
json_obj = JSON.parse(s);
} catch (_e) {
return false;
}
curr_path.push(new JSONDecodeRoute());
this.loopThroughJSON(json_obj, curr_path, hostname, req_id);
return true;
}
pushEntry(
entry: string,
curr_path: ValPath,
hostname: string,
req_id: number
) {
let rec = <HARRecord>{
hostname: hostname,
value: entry,
path: curr_path,
req_id,
importance: 0,
id: this.entries.length,
};
this.entries.push(rec);
}
parseEntry(
entry: string,
curr_path: ValPath,
hostname: string,
req_id: number
) {
this.pushEntry(entry, curr_path, hostname, req_id);
this.tryParseBase64(entry, [...curr_path], hostname, req_id);
this.tryParseUrlencodedParams(entry, [...curr_path], hostname, req_id);
this.tryParseJSON(entry, [...curr_path], hostname, req_id);
}
parseBinaryEntry(
entry: Uint8Array,
curr_path: ValPath,
hostname: string,
req_id: number
) {
this.pushEntry(
Hex.toHex(entry),
[...curr_path, new HexEncodeRoute()],
hostname,
req_id
);
this.tryParseBinaryToString(entry, curr_path, hostname, req_id);
this.tryParseDeflate(entry, curr_path, hostname, req_id);
}
public async try_parse(raw_har: Blob): Promise<null | Error> {
const har = await raw_har.text();
let har_obj: HAR.Har;
try {
har_obj = JSON.parse(har);
} catch (e) {
console.error(e);
return new Error("invalid HAR file, not in JSON format");
}
const har_obj_log = har_obj.log;
if (!har_obj_log)
return new Error("invalid HAR file, got no log section");
const har_entries = har_obj_log.entries;
this.har_entries = har_entries;
if (!har_entries || !Array.isArray(har_entries))
return new Error(
"invalid HAR file, got no correct log->etries section"
);
// from this point on let's assume everytihng is valid, and if not, just print a generic error
try {
for (let i = 0; i < har_entries.length; i++) {
const req = har_entries[i].request;
const url = new URL(req.url);
let hostname = this.get_real_domain(url.hostname);
this.parseURLParams(url.searchParams, [], hostname, i);
this.parseHeaders(req.headers, [], hostname, i);
const body_size = req.bodySize;
if (body_size !== 0 && req.postData && req.postData.text) {
let body = req.postData.text;
// TODO: entry.postData.encoding === 'base64', should not be part of the route, but often produces binary data, so carefull with that
// if (entry.postData.encoding === 'base64') {
// }
this.parseEntry(body, [new BodyRoute()], hostname, i);
}
}
} catch (e) {
return new Error(
"Failed while parsing the HAR file, got error: " + e
);
}
return null;
}
public process_private_information(private_info: [string, string][]) {
this.private_information = private_info;
-
- // let prev = this.kv.get(hostname);
- // if (prev) prev.push(rec);
- // else this.kv.set(hostname, [rec]);
-
+ this.summary = new Map();
for (let r of this.entries) {
r.private_info_idxs = [];
r.importance = 0;
let prev = this.summary.get(r.hostname);
if (!prev) {
prev = <SummaryRecord>{
private_info_idxs: new Set(),
importance: 0,
num_entries: 0,
num_entries_with_priv_info: 0,
};
this.summary.set(r.hostname, prev);
}
prev.num_entries++;
for (let [
idx,
[data, _desc],
] of this.private_information.entries()) {
if (r.value.includes(data)) {
r.private_info_idxs.push(idx);
r.importance +=
0.5 + data.length / r.value.length + 0.2 / r.path.length;
prev.private_info_idxs.add(idx);
}
}
if (r.private_info_idxs.length != 0)
prev.num_entries_with_priv_info++;
prev.importance += r.importance;
}
console.log("num values: ", this.entries.length);
}
}
diff --git a/src/template.html b/src/template.html
index 3da6e74..f9d4e85 100644
--- a/src/template.html
+++ b/src/template.html
@@ -1,173 +1,179 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link href="https://unpkg.com/tabulator-tables@6.3.1/dist/css/tabulator.min.css" rel="stylesheet">
</head>
<body>
<nav>
<a onclick="main.open_tab(event, 'tab_1')">Setup</a>
<a onclick="main.open_tab(event, 'tab_2')">Details</a>
<a onclick="main.open_tab(event, 'tab_3')">Summary</a>
</nav>
<main>
<div class="tab active" id="tab_1">
<form>
<input type="file" id="file"></input>
<div id="error_box"></div>
</form>
<div>
<table id="private-info-table">
<thead>
<tr>
<th>Information</th>
<th>Description</th>
</tr>
</thead>
<tbody id="private-info-table-body">
<tr>
<td>42</td>
<td>the real secret of life</td>
</tr>
<!-- PRIVATE INFO -->
</tbody>
</table>
<form id="private-info-form">
<input type="text" name="info" placeholder="New information"/>
<input type="text" name="desc" placeholder="Information description"/>
<button type="submit">Submit</button>
</form>
</div>
</div>
<div class="tab" id="tab_2">
<div id="tab_2_content">
<div id="table-wrapper">
<div id="main-table"></div>
</div>
<div id="expanded_info">
<div class="card">
<div class="card-label">Request Info</div>
<div class="card-content">
<div>
<span class="card-title">Request method: </span>
<span id="details_req_method"></span>
</div>
<div>
<span class="card-title">Request URL: </span>
<span id="details_req_url"></span>
</div>
</div>
</div>
<div class="card">
<div class="card-label">Path</div>
<div class="card-content">
<div id="curr_path"></div>
</div>
</div>
<div class="card">
<div class="card-label">Decoded msg</div>
<div class="card-content">
<textarea id="curr_decoded_msg" rows=5></textarea>
</div>
</div>
</div>
</div>
</div>
<div class="tab" id="tab_3">
<h1>Summary</h1>
<div>
<div id="domain_summary_table"></div>
</div>
</div>
</main>
</body>
</html>
<style>
html, body {
margin: 0;
height: 100vh;
display: flex;
flex-direction: column;
}
nav {
display: flex;
flex-direction: row;
align-items: center;
}
main {
flex-grow: 1;
display: flex;
flex-direction: column;
overflow-y: hidden;
}
nav>a {
padding: 0.5em;
border: 1px black solid;
text-align: center;
margin: 0;
display: inline-block;
box-sizing: border-box;
}
nav > a:hover {
background-color: darkgrey;
}
#table-wrapper {
width: 60%;
height: 100%;
}
#main-table {
height: 99%;
}
#expanded_info {
width: 40%;
overflow: scroll;
display: flex;
flex-direction: column;
gap: 5px;
}
#private-info-form {
display: flex;
flex-direction: row;
}
.tab {
display: none;
height: 100%;
}
.tab.active {
display: block;
}
#tab_2_content {
display: flex;
flex-direction:row;
height: 100%;
}
.card {
border: 1px solid black;
<!-- border-radius: 10px; -->
margin-left: 5px;
margin-right: 5px;
padding: 0;
overflow: hidden;
}
.card-content {
margin: 10px;
}
#curr_decoded_msg {
width: 100%;
resize: vertical;
box-sizing: border-box;
overflow-y: scroll;
}
.card-label {
width: 100%;
box-sizing: border-box;
font-weight: bold;
background-color: darkgrey;
padding: 0.5em;
}
#error_box > div {
background-color: red;
}
+ .delete_btn {
+ font-family: monospace;
+ font-weight: bold;
+ font-size: 1em;
+ text-align: center;
+ }
</style>
<!-- REPLACE ME -->
diff --git a/src/web_entrypoint.ts b/src/web_entrypoint.ts
index 92dbff5..aed0c3e 100644
--- a/src/web_entrypoint.ts
+++ b/src/web_entrypoint.ts
@@ -1,256 +1,262 @@
// make the export accessible from inline js
import * as main from "./web_entrypoint";
// for some reason doing the same with the window object doesn't work
(globalThis as any).main = main;
export function open_tab(evt: Event, tab_name: string) {
let i, tabcontent, tablinks;
// Get all elements with class="tab" and hide them
tabcontent = document.getElementsByClassName("tab");
for (i = 0; i < tabcontent.length; i++) {
if (tabcontent[i].id != tab_name) {
tabcontent[i].classList.remove("active");
} else {
tabcontent[i].classList.add("active");
}
}
// Get all elements with class="tablinks" and remove the class "active"
tablinks = document.getElementsByClassName("tablinks");
for (i = 0; i < tablinks.length; i++) {
tablinks[i].classList.remove("active");
}
// Show the current tab, and add an "active" class to the button that opened the tab
(evt.currentTarget as HTMLElement).classList.add("active");
}
import * as har_parser from "./index";
import { CellComponent, TabulatorFull as Tabulator } from "tabulator-tables";
let parser: har_parser.HARParser | undefined;
let filePicker = document.getElementById("file")!;
let curr_path = document.getElementById("curr_path")!;
let curr_decoded_msg = document.getElementById("curr_decoded_msg")! as HTMLTextAreaElement;
let details_req_method = document.getElementById("details_req_method")!;
let details_req_url = document.getElementById("details_req_url")!;
let error_box = document.getElementById("error_box")!;
filePicker.addEventListener("change", fullUpdate, false);
let privateInfoForm = <HTMLFormElement>(
document.getElementById("private-info-form")
);
privateInfoForm.addEventListener("submit", privateInfoSubmit, false);
let privateInfoTable = new Tabulator("#private-info-table", {
columns: [
+ { title: "", width: 20, formatter: function (cell, formatterParams) {
+ return `<div class="delete_btn">X</div>`
+ }, cellClick: async (_e, cell) => {
+ await cell.getRow().delete();
+ await updateTables();
+ }},
{ title: "Information", field: "information", editor: "input" },
{ title: "Description", field: "description", editor: "input" },
],
layout: "fitDataStretch",
});
privateInfoTable.on("cellEdited", updateTables);
function privateInfoSubmit(evt: SubmitEvent) {
evt.preventDefault();
let form_data = new FormData(privateInfoForm);
let info = form_data.get("info")!;
let desc = form_data.get("desc")!;
privateInfoTable.addRow({
information: info.toString(),
description: desc.toString(),
});
updateTables();
}
async function createParser(har: Blob) {
parser = new har_parser.HARParser();
let res = await parser.try_parse(har);
if (res === null) {
error_box.innerHTML = "";
await updateTables();
} else {
let div = document.createElement("div");
div.innerText = res.toString();
error_box.appendChild(div);
}
}
async function fullUpdate(evt: any) {
var files = evt.target.files;
var file = files[0];
await createParser(file);
}
function getPrivateInfo(): [string, string][] {
let ret: [string, string][] = [];
let data = privateInfoTable.getData();
for (const row of data) {
ret.push([row.information, row.description]);
}
return ret;
}
let main_table: Tabulator | undefined;
let summary_table: Tabulator | undefined;
let priv_info: [string, string][];
function createTables() {
if (!parser) return;
priv_info = getPrivateInfo();
parser.process_private_information(priv_info);
let priv_info_formatter = (cell: CellComponent, formatterParams: {}, onRendered: {}) => {
//cell - the cell component
//formatterParams - parameters set for the column
//onRendered - function to call when the formatter has been rendered
let cell_el = document.createElement("div");
for (let i of cell.getValue()) {
let btn = document.createElement("button");
btn.innerText = priv_info[i][1];
btn.addEventListener("click", (_) => {
cell.popup(
priv_info[i][1] + ": " + priv_info[i][0],
"center"
);
});
cell_el.appendChild(btn);
}
return cell_el;
};
main_table = new Tabulator("#main-table", {
data: parser.entries, //assign data to table
layout: "fitColumns", //fit columns to width of table (optional)
movableColumns: true, //allow column order to be changed
paginationCounter: "rows", //display count of paginated rows in footer
height: "99%",
columns: [
//Define Table Columns
{ title: "Domain", field: "hostname" },
{
title: "Path",
field: "path",
formatter: function (cell, formatterParams, onRendered) {
let val_path: har_parser.ValPath = cell.getValue();
let path = "";
for (const [idx, path_part] of val_path.entries()) {
path += path_part.pretty_print();
if (idx != val_path.length - 1) path += " -> ";
}
return path;
},
},
{ title: "Value", field: "value" },
{
title: "Private info found",
field: "private_info_idxs",
formatter: priv_info_formatter,
},
{ title: "Importance", field: "importance" },
],
rowFormatter: (row) => {
let data = row.getData();
row.getElement().addEventListener("click", (_ev) => {
_ev.stopPropagation();
_ev.stopImmediatePropagation();
let idx = row.getIndex();
curr_path.innerHTML = "";
for (let i = 0; i < parser!.entries[idx].path.length; i++) {
let arrow = document.createElement("span");
arrow.innerText = "->";
const val = parser!.entries[idx].path[i];
let a = document.createElement("a");
a.addEventListener("click", (ev) => {
ev.stopPropagation();
ev.stopImmediatePropagation();
});
a.innerText = val.pretty_print();
a.href = "javascript:void(0);";
curr_path.appendChild(a);
if (i < parser!.entries[idx].path.length - 1)
curr_path.appendChild(arrow);
}
curr_decoded_msg.value = data.value;
let req_id = parser!.entries[idx].req_id;
if (
!parser ||
!parser.har_entries ||
!parser.har_entries[req_id].request
)
return;
details_req_url.innerText =
parser.har_entries[req_id].request.url.toString();
details_req_method.innerText =
parser.har_entries[req_id].request.method.toString();
});
},
});
let summary_data = [];
for (let [domain, entry] of parser.summary.entries()) {
summary_data.push({domain, entry});
}
summary_table = new Tabulator("#domain_summary_table", {
data: summary_data, //assign data to table
layout: "fitColumns", //fit columns to width of table (optional)
movableColumns: true, //allow column order to be changed
paginationCounter: "rows", //display count of paginated rows in footer
columns: [
//Define Table Columns
{ title: "Domain", field: "domain" },
{ title: "Importance", field: "entry.importance" },
{ title: "Total Entries", field: "entry.num_entries" },
{ title: "Total with private info", field: "entry.num_entries_with_priv_info" },
{
title: "Private info found",
field: "entry.private_info_idxs",
formatter: priv_info_formatter,
},
]
});
}
async function updateTables() {
if (!parser)
return;
if (!main_table || !summary_table) {
createTables();
return;
}
priv_info = getPrivateInfo();
parser.process_private_information(priv_info);
await main_table.setData(parser.entries);
let summary_data = [];
for (let [domain, entry] of parser.summary.entries()) {
summary_data.push({domain, entry});
}
await summary_table.setData(summary_data);
}
let har_file = document.getElementById("har_file");
if (har_file) {
createParser(new Blob([har_file.innerText]));
}

File Metadata

Mime Type
text/x-diff
Expires
Sun, Nov 2, 17:46 (14 h, 32 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1012631
Default Alt Text
(24 KB)

Event Timeline