Page Menu
Home
Sealhub
Search
Configure Global Search
Log In
Files
F10352655
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
24 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/src/index.ts b/src/index.ts
index 0130d70..40fded7 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,523 +1,519 @@
import { list as suffix_list } from "./suffix_list.js";
import * as Base64 from "base64-js";
import * as Hex from "@smithy/util-hex-encoding";
import pako from "pako";
import * as HAR from "har-format";
export type ValPath = HARValRoute[];
interface HARValRoute {
pretty_print(): string;
}
export function val_path_cmp(v1: ValPath, v2: ValPath): boolean {
if (v1.length != v2.length) return false;
for (let i = 0; i < v1.length; i++) {
if (v1[i].pretty_print() !== v2[i].pretty_print()) return false;
}
return true;
}
export interface HARRecord {
id: number;
hostname: string;
path: ValPath;
value: string;
private_info_idxs: number[];
req_id: number;
importance: number;
}
export interface SummaryRecord {
private_info_idxs: Set<number>;
importance: number;
num_entries: number;
num_entries_with_priv_info: number;
}
interface HARHeader {
name: string;
value: string;
}
export class HeaderRoute implements HARValRoute {
type: "header";
name: string;
constructor(name: string) {
this.name = name;
}
pretty_print(): string {
return "Header('" + this.name + "')";
}
}
export class BodyRoute implements HARValRoute {
type: "body";
constructor() {}
pretty_print(): string {
return "Body";
}
}
export class URLParamRoute implements HARValRoute {
type: "urlparam";
name: string;
constructor(name: string) {
this.name = name;
}
pretty_print(): string {
return "UrlParam('" + this.name + "')";
}
}
export class SplitByRoute implements HARValRoute {
type: "splitby";
by: string;
constructor(by: string) {
this.by = by;
}
pretty_print(): string {
return "SplitBy('" + this.by + "')";
}
}
export class JSONDecodeRoute implements HARValRoute {
type: "jsondecode";
constructor() {}
pretty_print(): string {
return "JSONDecode()";
}
}
export class HexEncodeRoute implements HARValRoute {
type: "hexencode";
constructor() {}
pretty_print(): string {
return "HexEncode()";
}
}
export class IdxRoute implements HARValRoute {
type: "idx";
idx: number;
constructor(idx: number) {
this.idx = idx;
}
pretty_print(): string {
return "Idx('" + this.idx + "')";
}
}
export class KeyRoute implements HARValRoute {
type: "key";
key: string;
constructor(key: string) {
this.key = key;
}
pretty_print(): string {
return "Key('" + this.key + "')";
}
}
export class Base64DecodeRoute implements HARValRoute {
type: "unbase64";
constructor() {}
pretty_print(): string {
return "Base64Decode()";
}
}
export class UnbinaryString implements HARValRoute {
type: "unbinary_str";
constructor() {}
pretty_print(): string {
return "BinaryToString()";
}
}
export class InflateRoute implements HARValRoute {
type: "inflate";
constructor() {}
pretty_print(): string {
return "Inflate()";
}
}
export class HARParser {
har_entries: HAR.Entry[] | undefined;
entries: HARRecord[] = [];
summary: Map<String, SummaryRecord> = new Map();
suffixes: Set<String>;
// data -> desc
private_information: [string, string][];
curr_id: number = 0;
constructor() {
this.suffixes = new Set(suffix_list);
}
get_real_domain(host: string): string {
let subdomains = host.split(".");
let i = 1;
let curr_string;
for (; i <= subdomains.length; i++) {
curr_string = "";
let host_sections = subdomains.slice(subdomains.length - i);
for (const [idx, s] of host_sections.entries()) {
curr_string += s;
if (idx != host_sections.length - 1) curr_string += ".";
}
if (!this.suffixes.has(curr_string)) break;
}
if (curr_string === undefined)
throw new Error("failed to get a domain from: " + host);
return curr_string;
}
parseHeaders(
headers: HARHeader[],
curr_path: ValPath,
hostname: string,
req_id: number
) {
for (const { name, value } of headers) {
let new_path = [...curr_path, new HeaderRoute(name)];
this.parseEntry(value, new_path, hostname, req_id);
}
}
parseURLParams(
params: URLSearchParams,
curr_path: ValPath,
hostname: string,
req_id: number
) {
let i = 0;
for (const [key, value] of params.entries()) {
let new_path: ValPath = [...curr_path, new URLParamRoute(key)];
this.parseEntry(value, new_path, hostname, req_id);
new_path = [...curr_path, new IdxRoute(i)];
this.parseEntry(key, new_path, hostname, req_id);
i++;
}
}
tryParseUrlencodedParams(
s: string,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
let decoded_everything = true;
s.split("&")
.filter((e) => {
if (e.indexOf("=") !== -1) return true;
decoded_everything = false;
return false;
})
.entries()
.forEach(([idx, e]) => {
let i = e.indexOf("=");
let k;
let v;
try {
v = decodeURIComponent(e.slice(i + 1).replace(/\+/g, " "));
k = decodeURIComponent(e.slice(0, i).replace(/\+/g, " "));
} catch (_) {
decoded_everything = false;
}
if (v && k) {
let new_path = [...curr_path, new URLParamRoute(k)];
this.parseEntry(v, new_path, hostname, req_id);
new_path = [...curr_path, new IdxRoute(idx)];
this.parseEntry(k, new_path, hostname, req_id);
}
});
return decoded_everything;
}
tryParseDeflate(
s: Uint8Array,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
try {
let result = pako.inflate(s);
this.parseBinaryEntry(
result,
[...curr_path, new InflateRoute()],
hostname,
req_id
);
} catch (e) {
return false;
}
return true;
}
tryParseBinaryToString(
entry: Uint8Array,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
try {
let decoded_str = new TextDecoder("utf8", {
fatal: true,
}).decode(entry);
this.parseEntry(
decoded_str,
[...curr_path, new UnbinaryString()],
hostname,
req_id
);
} catch (_) {
return false;
}
return true;
}
tryParseBase64(
s: string,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
if (s.length == 0) return false;
let decoded_str: string = "";
let firt_non_base64 =
/([^ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+=\/])/;
let contains_at_least_2 =
/([^ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+=\/]).*([^ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+=\/])/;
if (contains_at_least_2.test(s)) return false;
let matches = firt_non_base64.exec(s);
let matches_uniq = new Set(matches);
let is_base64 = matches_uniq.size === 0;
if (is_base64) {
try {
// works for the browser
let buff = Base64.toByteArray(s);
this.parseBinaryEntry(
buff,
[...curr_path, new Base64DecodeRoute()],
hostname,
req_id
);
} catch (_) {
is_base64 = false;
}
} else {
if (!matches_uniq) return false;
if (matches_uniq.size != 1) return false;
let everything_decoded = true;
let split_char: string = matches_uniq.values().next().value;
let substrs = s.split(split_char);
curr_path.push(new SplitByRoute(split_char));
let idx = 0;
for (const substr of substrs) {
if (substr.length == 0) continue;
let new_path = [...curr_path, new IdxRoute(idx)];
everything_decoded =
this.tryParseBase64(substr, new_path, hostname, req_id) &&
everything_decoded;
idx++;
}
return everything_decoded;
}
if (decoded_str.length == 0) return false;
curr_path.push(new Base64DecodeRoute());
this.parseEntry(decoded_str, curr_path, hostname, req_id);
return true;
}
loopThroughJSON(
obj: any,
curr_path: ValPath,
hostname: string,
req_id: number
) {
let i = 0;
for (let key in obj) {
let path_idx = [...curr_path, new IdxRoute(i)];
this.parseEntry(key, path_idx, hostname, req_id);
let path_key = [...curr_path, new KeyRoute(key)];
if (typeof obj[key] === "object") {
if (Array.isArray(obj[key])) {
for (let i = 0; i < obj[key].length; i++) {
let new_path = [...path_key, new IdxRoute(i)];
this.loopThroughJSON(
obj[key][i],
new_path,
hostname,
req_id
);
}
} else {
this.loopThroughJSON(obj[key], path_key, hostname, req_id);
}
} else {
this.parseEntry(obj[key] + "", path_key, hostname, req_id);
}
}
i++;
}
tryParseJSON(
s: string,
curr_path: ValPath,
hostname: string,
req_id: number
): boolean {
if (s.length == 0) return false;
let json_obj;
try {
json_obj = JSON.parse(s);
} catch (_e) {
return false;
}
curr_path.push(new JSONDecodeRoute());
this.loopThroughJSON(json_obj, curr_path, hostname, req_id);
return true;
}
pushEntry(
entry: string,
curr_path: ValPath,
hostname: string,
req_id: number
) {
let rec = <HARRecord>{
hostname: hostname,
value: entry,
path: curr_path,
req_id,
importance: 0,
id: this.entries.length,
};
this.entries.push(rec);
}
parseEntry(
entry: string,
curr_path: ValPath,
hostname: string,
req_id: number
) {
this.pushEntry(entry, curr_path, hostname, req_id);
this.tryParseBase64(entry, [...curr_path], hostname, req_id);
this.tryParseUrlencodedParams(entry, [...curr_path], hostname, req_id);
this.tryParseJSON(entry, [...curr_path], hostname, req_id);
}
parseBinaryEntry(
entry: Uint8Array,
curr_path: ValPath,
hostname: string,
req_id: number
) {
this.pushEntry(
Hex.toHex(entry),
[...curr_path, new HexEncodeRoute()],
hostname,
req_id
);
this.tryParseBinaryToString(entry, curr_path, hostname, req_id);
this.tryParseDeflate(entry, curr_path, hostname, req_id);
}
public async try_parse(raw_har: Blob): Promise<null | Error> {
const har = await raw_har.text();
let har_obj: HAR.Har;
try {
har_obj = JSON.parse(har);
} catch (e) {
console.error(e);
return new Error("invalid HAR file, not in JSON format");
}
const har_obj_log = har_obj.log;
if (!har_obj_log)
return new Error("invalid HAR file, got no log section");
const har_entries = har_obj_log.entries;
this.har_entries = har_entries;
if (!har_entries || !Array.isArray(har_entries))
return new Error(
"invalid HAR file, got no correct log->etries section"
);
// from this point on let's assume everytihng is valid, and if not, just print a generic error
try {
for (let i = 0; i < har_entries.length; i++) {
const req = har_entries[i].request;
const url = new URL(req.url);
let hostname = this.get_real_domain(url.hostname);
this.parseURLParams(url.searchParams, [], hostname, i);
this.parseHeaders(req.headers, [], hostname, i);
const body_size = req.bodySize;
if (body_size !== 0 && req.postData && req.postData.text) {
let body = req.postData.text;
// TODO: entry.postData.encoding === 'base64', should not be part of the route, but often produces binary data, so carefull with that
// if (entry.postData.encoding === 'base64') {
// }
this.parseEntry(body, [new BodyRoute()], hostname, i);
}
}
} catch (e) {
return new Error(
"Failed while parsing the HAR file, got error: " + e
);
}
return null;
}
public process_private_information(private_info: [string, string][]) {
this.private_information = private_info;
-
- // let prev = this.kv.get(hostname);
- // if (prev) prev.push(rec);
- // else this.kv.set(hostname, [rec]);
-
+ this.summary = new Map();
for (let r of this.entries) {
r.private_info_idxs = [];
r.importance = 0;
let prev = this.summary.get(r.hostname);
if (!prev) {
prev = <SummaryRecord>{
private_info_idxs: new Set(),
importance: 0,
num_entries: 0,
num_entries_with_priv_info: 0,
};
this.summary.set(r.hostname, prev);
}
prev.num_entries++;
for (let [
idx,
[data, _desc],
] of this.private_information.entries()) {
if (r.value.includes(data)) {
r.private_info_idxs.push(idx);
r.importance +=
0.5 + data.length / r.value.length + 0.2 / r.path.length;
prev.private_info_idxs.add(idx);
}
}
if (r.private_info_idxs.length != 0)
prev.num_entries_with_priv_info++;
prev.importance += r.importance;
}
console.log("num values: ", this.entries.length);
}
}
diff --git a/src/template.html b/src/template.html
index 3da6e74..f9d4e85 100644
--- a/src/template.html
+++ b/src/template.html
@@ -1,173 +1,179 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link href="https://unpkg.com/tabulator-tables@6.3.1/dist/css/tabulator.min.css" rel="stylesheet">
</head>
<body>
<nav>
<a onclick="main.open_tab(event, 'tab_1')">Setup</a>
<a onclick="main.open_tab(event, 'tab_2')">Details</a>
<a onclick="main.open_tab(event, 'tab_3')">Summary</a>
</nav>
<main>
<div class="tab active" id="tab_1">
<form>
<input type="file" id="file"></input>
<div id="error_box"></div>
</form>
<div>
<table id="private-info-table">
<thead>
<tr>
<th>Information</th>
<th>Description</th>
</tr>
</thead>
<tbody id="private-info-table-body">
<tr>
<td>42</td>
<td>the real secret of life</td>
</tr>
<!-- PRIVATE INFO -->
</tbody>
</table>
<form id="private-info-form">
<input type="text" name="info" placeholder="New information"/>
<input type="text" name="desc" placeholder="Information description"/>
<button type="submit">Submit</button>
</form>
</div>
</div>
<div class="tab" id="tab_2">
<div id="tab_2_content">
<div id="table-wrapper">
<div id="main-table"></div>
</div>
<div id="expanded_info">
<div class="card">
<div class="card-label">Request Info</div>
<div class="card-content">
<div>
<span class="card-title">Request method: </span>
<span id="details_req_method"></span>
</div>
<div>
<span class="card-title">Request URL: </span>
<span id="details_req_url"></span>
</div>
</div>
</div>
<div class="card">
<div class="card-label">Path</div>
<div class="card-content">
<div id="curr_path"></div>
</div>
</div>
<div class="card">
<div class="card-label">Decoded msg</div>
<div class="card-content">
<textarea id="curr_decoded_msg" rows=5></textarea>
</div>
</div>
</div>
</div>
</div>
<div class="tab" id="tab_3">
<h1>Summary</h1>
<div>
<div id="domain_summary_table"></div>
</div>
</div>
</main>
</body>
</html>
<style>
html, body {
margin: 0;
height: 100vh;
display: flex;
flex-direction: column;
}
nav {
display: flex;
flex-direction: row;
align-items: center;
}
main {
flex-grow: 1;
display: flex;
flex-direction: column;
overflow-y: hidden;
}
nav>a {
padding: 0.5em;
border: 1px black solid;
text-align: center;
margin: 0;
display: inline-block;
box-sizing: border-box;
}
nav > a:hover {
background-color: darkgrey;
}
#table-wrapper {
width: 60%;
height: 100%;
}
#main-table {
height: 99%;
}
#expanded_info {
width: 40%;
overflow: scroll;
display: flex;
flex-direction: column;
gap: 5px;
}
#private-info-form {
display: flex;
flex-direction: row;
}
.tab {
display: none;
height: 100%;
}
.tab.active {
display: block;
}
#tab_2_content {
display: flex;
flex-direction:row;
height: 100%;
}
.card {
border: 1px solid black;
<!-- border-radius: 10px; -->
margin-left: 5px;
margin-right: 5px;
padding: 0;
overflow: hidden;
}
.card-content {
margin: 10px;
}
#curr_decoded_msg {
width: 100%;
resize: vertical;
box-sizing: border-box;
overflow-y: scroll;
}
.card-label {
width: 100%;
box-sizing: border-box;
font-weight: bold;
background-color: darkgrey;
padding: 0.5em;
}
#error_box > div {
background-color: red;
}
+ .delete_btn {
+ font-family: monospace;
+ font-weight: bold;
+ font-size: 1em;
+ text-align: center;
+ }
</style>
<!-- REPLACE ME -->
diff --git a/src/web_entrypoint.ts b/src/web_entrypoint.ts
index 92dbff5..aed0c3e 100644
--- a/src/web_entrypoint.ts
+++ b/src/web_entrypoint.ts
@@ -1,256 +1,262 @@
// make the export accessible from inline js
import * as main from "./web_entrypoint";
// for some reason doing the same with the window object doesn't work
(globalThis as any).main = main;
export function open_tab(evt: Event, tab_name: string) {
let i, tabcontent, tablinks;
// Get all elements with class="tab" and hide them
tabcontent = document.getElementsByClassName("tab");
for (i = 0; i < tabcontent.length; i++) {
if (tabcontent[i].id != tab_name) {
tabcontent[i].classList.remove("active");
} else {
tabcontent[i].classList.add("active");
}
}
// Get all elements with class="tablinks" and remove the class "active"
tablinks = document.getElementsByClassName("tablinks");
for (i = 0; i < tablinks.length; i++) {
tablinks[i].classList.remove("active");
}
// Show the current tab, and add an "active" class to the button that opened the tab
(evt.currentTarget as HTMLElement).classList.add("active");
}
import * as har_parser from "./index";
import { CellComponent, TabulatorFull as Tabulator } from "tabulator-tables";
let parser: har_parser.HARParser | undefined;
let filePicker = document.getElementById("file")!;
let curr_path = document.getElementById("curr_path")!;
let curr_decoded_msg = document.getElementById("curr_decoded_msg")! as HTMLTextAreaElement;
let details_req_method = document.getElementById("details_req_method")!;
let details_req_url = document.getElementById("details_req_url")!;
let error_box = document.getElementById("error_box")!;
filePicker.addEventListener("change", fullUpdate, false);
let privateInfoForm = <HTMLFormElement>(
document.getElementById("private-info-form")
);
privateInfoForm.addEventListener("submit", privateInfoSubmit, false);
let privateInfoTable = new Tabulator("#private-info-table", {
columns: [
+ { title: "", width: 20, formatter: function (cell, formatterParams) {
+ return `<div class="delete_btn">X</div>`
+ }, cellClick: async (_e, cell) => {
+ await cell.getRow().delete();
+ await updateTables();
+ }},
{ title: "Information", field: "information", editor: "input" },
{ title: "Description", field: "description", editor: "input" },
],
layout: "fitDataStretch",
});
privateInfoTable.on("cellEdited", updateTables);
function privateInfoSubmit(evt: SubmitEvent) {
evt.preventDefault();
let form_data = new FormData(privateInfoForm);
let info = form_data.get("info")!;
let desc = form_data.get("desc")!;
privateInfoTable.addRow({
information: info.toString(),
description: desc.toString(),
});
updateTables();
}
async function createParser(har: Blob) {
parser = new har_parser.HARParser();
let res = await parser.try_parse(har);
if (res === null) {
error_box.innerHTML = "";
await updateTables();
} else {
let div = document.createElement("div");
div.innerText = res.toString();
error_box.appendChild(div);
}
}
async function fullUpdate(evt: any) {
var files = evt.target.files;
var file = files[0];
await createParser(file);
}
function getPrivateInfo(): [string, string][] {
let ret: [string, string][] = [];
let data = privateInfoTable.getData();
for (const row of data) {
ret.push([row.information, row.description]);
}
return ret;
}
let main_table: Tabulator | undefined;
let summary_table: Tabulator | undefined;
let priv_info: [string, string][];
function createTables() {
if (!parser) return;
priv_info = getPrivateInfo();
parser.process_private_information(priv_info);
let priv_info_formatter = (cell: CellComponent, formatterParams: {}, onRendered: {}) => {
//cell - the cell component
//formatterParams - parameters set for the column
//onRendered - function to call when the formatter has been rendered
let cell_el = document.createElement("div");
for (let i of cell.getValue()) {
let btn = document.createElement("button");
btn.innerText = priv_info[i][1];
btn.addEventListener("click", (_) => {
cell.popup(
priv_info[i][1] + ": " + priv_info[i][0],
"center"
);
});
cell_el.appendChild(btn);
}
return cell_el;
};
main_table = new Tabulator("#main-table", {
data: parser.entries, //assign data to table
layout: "fitColumns", //fit columns to width of table (optional)
movableColumns: true, //allow column order to be changed
paginationCounter: "rows", //display count of paginated rows in footer
height: "99%",
columns: [
//Define Table Columns
{ title: "Domain", field: "hostname" },
{
title: "Path",
field: "path",
formatter: function (cell, formatterParams, onRendered) {
let val_path: har_parser.ValPath = cell.getValue();
let path = "";
for (const [idx, path_part] of val_path.entries()) {
path += path_part.pretty_print();
if (idx != val_path.length - 1) path += " -> ";
}
return path;
},
},
{ title: "Value", field: "value" },
{
title: "Private info found",
field: "private_info_idxs",
formatter: priv_info_formatter,
},
{ title: "Importance", field: "importance" },
],
rowFormatter: (row) => {
let data = row.getData();
row.getElement().addEventListener("click", (_ev) => {
_ev.stopPropagation();
_ev.stopImmediatePropagation();
let idx = row.getIndex();
curr_path.innerHTML = "";
for (let i = 0; i < parser!.entries[idx].path.length; i++) {
let arrow = document.createElement("span");
arrow.innerText = "->";
const val = parser!.entries[idx].path[i];
let a = document.createElement("a");
a.addEventListener("click", (ev) => {
ev.stopPropagation();
ev.stopImmediatePropagation();
});
a.innerText = val.pretty_print();
a.href = "javascript:void(0);";
curr_path.appendChild(a);
if (i < parser!.entries[idx].path.length - 1)
curr_path.appendChild(arrow);
}
curr_decoded_msg.value = data.value;
let req_id = parser!.entries[idx].req_id;
if (
!parser ||
!parser.har_entries ||
!parser.har_entries[req_id].request
)
return;
details_req_url.innerText =
parser.har_entries[req_id].request.url.toString();
details_req_method.innerText =
parser.har_entries[req_id].request.method.toString();
});
},
});
let summary_data = [];
for (let [domain, entry] of parser.summary.entries()) {
summary_data.push({domain, entry});
}
summary_table = new Tabulator("#domain_summary_table", {
data: summary_data, //assign data to table
layout: "fitColumns", //fit columns to width of table (optional)
movableColumns: true, //allow column order to be changed
paginationCounter: "rows", //display count of paginated rows in footer
columns: [
//Define Table Columns
{ title: "Domain", field: "domain" },
{ title: "Importance", field: "entry.importance" },
{ title: "Total Entries", field: "entry.num_entries" },
{ title: "Total with private info", field: "entry.num_entries_with_priv_info" },
{
title: "Private info found",
field: "entry.private_info_idxs",
formatter: priv_info_formatter,
},
]
});
}
async function updateTables() {
if (!parser)
return;
if (!main_table || !summary_table) {
createTables();
return;
}
priv_info = getPrivateInfo();
parser.process_private_information(priv_info);
await main_table.setData(parser.entries);
let summary_data = [];
for (let [domain, entry] of parser.summary.entries()) {
summary_data.push({domain, entry});
}
await summary_table.setData(summary_data);
}
let har_file = document.getElementById("har_file");
if (har_file) {
createParser(new Blob([har_file.innerText]));
}
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Sun, Nov 2, 17:46 (14 h, 32 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1012631
Default Alt Text
(24 KB)
Attached To
Mode
R171 har-analyzer
Attached
Detach File
Event Timeline
Log In to Comment