hectic-lab/util.nix
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(package): hemar: json_escape()

Browse Source
This commit is contained in:
yukkop
2025-12-09 03:47:27 +00:00
parent 71082b6414
commit 73dc9aee9e
19 changed files with 264 additions and 843 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
package/hemar/parser/default.nix
View File

@@ -6,19 +6,6 @@ let
"nounset"
];
test = hectic.writeShellApplication {
inherit shell bashOptions;
name = "hemar-test";
runtimeInputs = [ ];
text = ''
# shellcheck disable=SC2034
WORKSPACE=${./.}
${builtins.readFile hectic.helpers.posix-shell.log}
${builtins.readFile ./test.sh}
'';
};
hemar = hectic.writeShellApplication {
inherit shell bashOptions;
name = "hemar";
@@ -34,5 +21,5 @@ let
in
symlinkJoin {
name = "hemar";
paths = [ hemar test ];
paths = [ hemar ];
}

235
package/hemar/parser/hemar.sh
View File

@@ -1,7 +1,5 @@
#!/bin/dash
log notice "running"
# segmented-path
# segment
# Syntax scheme:
@@ -97,57 +95,45 @@ log notice "running"
# '{' '0020' . '10FFFF' - '['
# AST Plex:
# AST Structure:
#
# Type = 0..=5
# The parser outputs a JSON array of elements directly (not wrapped in an object).
#
# Text = string # just a text body
# Element types (currently implemented):
#
# Interpolation = string # path to variable
# Text = {
# "type": "text",
# "value": string # text content
# }
#
# Interpolation = {
# "type": "interpolation",
# "path": string # path to variable in data model
# }
#
# Element types (planned for MVP):
#
# Section = {
# v = string # item variable name for loop
# p = string # path to array for iteration
# b = [Element] # section body
#
# "type": "section",
# "variable": string # item variable name for loop
# "path": string # path to array for iteration
# "body": [Element] # section body (nested elements)
# }
#
# End = null
# Element types (planned for future, not MVP):
#
# Include = {
# "type": "include",
# "path": string # path to template file to include
# }
#
# Compute = {
# l = string # programing language
# b = string # function body
# "type": "compute",
# "language": string # programming language (dash, plpgsql, etc.)
# "body": string # function body
# }
#
# Element = {
# t = Type # element type
# b = Text # element body
# | Interpolation
# | Section
# | End
# | Include
# | Compute
# }
#
# AbstarctSyntaxTree (ATS) = {
# e = [Element] # elements array
# }
AST=$(mktemp)
AST_key='.'
trap 'rm -f "$AST"' EXIT INT HUP
yq -o j -i '.' "$AST"
log debug "AST path: ${WHITE}${AST}"
# 0 - text
# 1 - deside tag type
# 2 - interpolation
# 3 - section
# 4 - include
# 5 - compute
STAGE=0
# AbstractSyntaxTree = [Element, ...] # array of elements
# is_ws(char) -> bool
is_ws() {
@@ -187,15 +173,38 @@ buf_reset() {
CURRENT_STAGE_BUFFER="$STAGE_BUFFER_1"
}
STAGE_BUFFER_1="$(mktemp)"
CURRENT_STAGE_BUFFER=$STAGE_BUFFER_1
trap 'rm -f "$STAGE_BUFFER_1"' EXIT INT HUP
log debug "stage buffer 1: ${WHITE}$STAGE_BUFFER_1"
# json_escape(value) -> str
json_escape() {
# TODO: escape functionality
printf '%s' "${1}" | sed 's/"/\\"/g'
local input="${1}"
local output=""
local char hex
while [ -n "$input" ]; do
char="${input%"${input#?}"}" # Get first character
input="${input#?}" # Remove first character
hex=$(printf '%d' "'$char")
case "$hex" in
34) output="${output}\\\"" ;; # "
92) output="${output}\\\\" ;; # \
10) output="${output}\\n" ;; # \n (newline)
13) output="${output}\\r" ;; # \r (carriage return)
9) output="${output}\\t" ;; # \t (tab)
8) output="${output}\\b" ;; # \b (backspace)
12) output="${output}\\f" ;; # \f (form feed)
*)
# NOTE(yukkop): escape control characters if they are not in the range 0x20-0x7E
if [ "$hex" -lt 32 ]; then
output="${output}\\u$(printf '%04x' "$hex")"
else
output="${output}${char}"
fi
;;
esac
done
printf '%s' "$output"
}
# finds close pattern and store the char to the stage buffers separating by spaces
@@ -489,67 +498,93 @@ parse() {
esac
}
while [ $# -gt 0 ]; do
case $1 in
-c|--compact-output)
OUTPUT_ARGS="${OUTPUT_ARGS+$OUTPUT_ARGS }-I=0"
shift
;;
--*|-*)
log error "argument $1 does not exists"
exit 9
;;
*)
log error "subcommand $1 does not exists"
exit 9
;;
esac
done
CHAR_N=1
LINE_N=1
if [ -z "${AS_LIBRARY+x}" ]; then
log notice "running"
while :; do
hex="$(dd bs=1 count=1 2>/dev/null | od -An -t u1)"
AST=$(mktemp)
AST_key='.'
trap 'rm -f "$AST"' EXIT INT HUP
[ -z "$hex" ] && {
break
}
yq -o j -i '.' "$AST"
# shellcheck disable=SC2059
char="$(printf "\\$(printf '%03o' "$hex")")"
log debug "AST path: ${WHITE}${AST}"
# NOTE: if $char is empty, it because `dd` returned '\n' but `$(...)`
# removed it as trailing '\n', so I set $char as '\n' here
[ -z "$char" ] && {
LINE_N=$((LINE_N+1))
char='
# 0 - text
# 1 - deside tag type
# 2 - interpolation
# 3 - section
# 4 - include
# 5 - compute
STAGE=0
STAGE_BUFFER_1="$(mktemp)"
CURRENT_STAGE_BUFFER=$STAGE_BUFFER_1
trap 'rm -f "$STAGE_BUFFER_1"' EXIT INT HUP
log debug "stage buffer 1: ${WHITE}$STAGE_BUFFER_1"
while [ $# -gt 0 ]; do
case $1 in
-c|--compact-output)
OUTPUT_ARGS="${OUTPUT_ARGS+$OUTPUT_ARGS }-I=0"
shift
;;
--*|-*)
log error "argument $1 does not exists"
exit 9
;;
*)
log error "subcommand $1 does not exists"
exit 9
;;
esac
done
CHAR_N=1
LINE_N=1
while :; do
hex="$(dd bs=1 count=1 2>/dev/null | od -An -t u1)"
[ -z "$hex" ] && {
break
}
# shellcheck disable=SC2059
char="$(printf "\\$(printf '%03o' "$hex")")"
# NOTE: if $char is empty, it because `dd` returned '\n' but `$(...)`
# removed it as trailing '\n', so I set $char as '\n' here
[ -z "$char" ] && {
LINE_N=$((LINE_N+1))
char='
'
}
}
log trace "char: $WHITE$char"
log trace "char: $WHITE$char"
parse "${char:?}"
parse "${char:?}"
CHAR_N=$((CHAR_N+1))
done
CHAR_N=$((CHAR_N+1))
done
log debug 'finishing'
log debug 'finishing'
# finish TEXT tag if file ends on it
if [ "$STAGE" -eq 0 ]; then
if [ "${open_tag_flag+x}" ]; then
unset open_tag_flag
printf '{' >> "$STAGE_BUFFER_1"
fi
# finish TEXT tag if file ends on it
if [ "$STAGE" -eq 0 ]; then
if [ "${open_tag_flag+x}" ]; then
unset open_tag_flag
printf '{' >> "$STAGE_BUFFER_1"
fi
buf=$(cat "$STAGE_BUFFER_1")
yq -o j -i "$AST_key += [{
\"type\": \"text\",
\"value\": \"$(json_escape "$buf")\"
}]" "$AST"
fi
buf=$(cat "$STAGE_BUFFER_1")
yq -o j -i "$AST_key += [{
\"type\": \"text\",
\"value\": \"$(json_escape "$buf")\"
}]" "$AST"
fi
# return the output
# shellcheck disable=SC2086
yq ${OUTPUT_ARGS:-} -o j "$AST"
# return the output
# shellcheck disable=SC2086
yq ${OUTPUT_ARGS:-} -o j "$AST"
fi

70
package/hemar/parser/src/plex/backend/env_var.sh
View File

@@ -1,70 +0,0 @@
#!/bin/dash
plex_set() {
local structname key val regex base esc_key regex esc temp
structname=$1 key=$2 val=$3
# construct regex for ancestors
regex="^$key="
base=$key
while expr "$base" : '.*\.' >/dev/null; do
base=$(printf '%s\n' "$base" | sed 's/\.[^.]*$//')
esc=$(printf '%s\n' "$base" | sed 's/\./\\./g')
regex="$regex|^$esc="
done
# add descendants
esc_key="$(printf '%s\n' "$key" | sed 's/\./\\./g')"
regex="$regex|^${esc_key}\."
# remove old
# <plex>=$(printf '%s\n' "$<plex>" | grep -v -E "$regex")
temp="$(eval "printf '%s\\n' \"\$$structname\"" | grep -v -E "$regex")"
eval "$structname=\"\$temp\""
# add new
eval "$structname=\$(printf '%s\\n%s=%s\\n' \"\$$structname\" \"\$key\" \"\$val\")"
}
plex_child() {
local structname prefix
structname=$1 prefix=$2
eval printf '%s\\n' \"\$"$structname"\" \
| grep "^$prefix\." \
| sed "s|^$prefix\.||"
}
plex_val() {
local structname key
structname=$1 key=$2
eval printf '%s\n' \"\$"$structname"\" | grep "^$key=" | cut -d= -f2-
}
plex_fetch() {
local structname key
structname=$1 key=$2
if eval printf '%s\\n' \"\$"$structname"\" | grep -q "^$key="; then
eval printf '%s\\n' \"\$"$structname"\" | grep "^$key=" | cut -d= -f2-
else
eval printf '%s\\n' \"\$"$structname"\" | grep "^$key\." | sed "s|^$key\.||"
fi
}
plex_push() {
local structname prefix val max idx newidx kv
structname=${1:?}
prefix=${2:?}
val=${3:?}
# find max index
max=0
for kv in $(plex_fetch "$structname" "$prefix"); do
idx=${kv%%=*}
[ "$idx" -gt "$max" ] 2>/dev/null && max=$idx
done
newidx=$((max + 1))
plex_set "$structname" "$prefix.$newidx" "$val"
}

50
package/hemar/parser/src/plex/backend/file.sh
View File

@@ -1,50 +0,0 @@
#!/bin/dash
PLEX_TEMP="$(mktemp -d)"
#plex_set(name, key, value)
plex_set() {
local plexfile key val regex base esc_key esc
plexfile="${PLEX_TEMP:?}${1:?}" key="${2:?}" val="${3:?}"
find PLEX_
}
plex_child() {
local plexfile key
plexfile="${PLEX_TEMP:?}${1:?}" key="${2:?}"
grep "^$key\." "" | sed "s|^$key\.||"
}
plex_val() {
local plexfile key
plexfile="${PLEX_TEMP:?}${1:?}" key="${2:?}"
grep "^$key=" | cut -d= -f2- "$plexfile"
}
plex_fetch() {
local plexfile key temp
plexfile="${PLEX_TEMP:?}${1:?}" key="${2:?}"
if temp="$(grep "^$key=" | cut -d= -f2- "$plexfile")"; then
printf '%s' "$temp"
else
grep "^$key\." "" | sed "s|^$key\.||"
fi
}
plex_push() {
local plex prefix val max idx newidx kv
plex="${1:?}" prefix="${2:?}" val="${3:?}"
# find max index
max=0
for kv in $(plex_fetch "$plex" "$prefix"); do
idx=${kv%%=*}
[ "$idx" -gt "$max" ] 2>/dev/null && max=$idx
done
newidx=$((max + 1))
plex_set "$plex" "$prefix.$newidx" "$val"
}

40
package/hemar/parser/src/plex/backend/yq-go.sh
View File

@@ -1,40 +0,0 @@
#!/bin/dash
PLEX_TEMP="$(mktemp -d)"
trap 'rm -rf $PLEX_TEMP' EXIT
#plex_set(name, key, value)
plex_set() {
local plexfile key val
plexfile="${PLEX_TEMP:?}/${1:?}.json" key="${2:?}" val="${3:?}"
touch "$plexfile"
yq -i ".$key = \"$val\"" "$plexfile"
}
#plex_child(name, key)
plex_child() {
plex_fetch "${1:?}" "${2:?}"
}
#plex_val(name, key)
plex_val() {
plex_fetch "${1:?}" "${2:?}"
}
#plex_val(name, key)
plex_fetch() {
local plexfile key
plexfile="${PLEX_TEMP:?}/${1:?}.json" key="${2:?}"
yq -r ".$key" "$plexfile"
}
#plex_push(name, prefix, val)
plex_push() {
local plexfile prefix val
plexfile="${PLEX_TEMP:?}/${1:?}.json" prefix="${2:?}" val="${3:?}"
yq -i ".$prefix += [\"$val\"]" "$plexfile"
}

21
package/hemar/parser/src/plex/plex.sh
View File

@@ -1,21 +0,0 @@
#!/bin/dash
init_plex() {
local backend
backend=${1:?}
case "$backend" in
env)
. ${WORKSPACE}/src/plex/backend/env_var.sh
;;
file)
. ${WORKSPACE}/src/plex/backend/file.sh
;;
yq-go)
. ${WORKSPACE}/src/plex/backend/yq-go.sh
;;
*)
exit 1
;;
esac
}

6
package/hemar/parser/test.sh
View File

@@ -1,6 +0,0 @@
#!/bin/dash
# shellcheck disable=SC1091
#. "${WORKSPACE:?}/test/plex/jq_backend_time.sh"
#. "${WORKSPACE:?}/test/plex/env_backend_time.sh"
. "${WORKSPACE:?}/test/plex/jq_backend.sh"

112
package/hemar/parser/test/plex/env_backend_time.sh
View File

@@ -1,112 +0,0 @@
#!/bin/dash
. "${WORKSPACE:?}/src/plex/plex.sh"
init_plex env
MY_STRUCT=''
math() {
awk "BEGIN {print $1}"
}
elapsed() {
local task time count decrease avg
task=$1
time=$2
count=$3
decrease=${4:-0}
avg=$(math "$time/$count-$decrease")
printf '\n[%s]\nelapsed %s seconds\n%s per second\n' \
"$task" "$avg" "$(math "1/$avg")" >&2
printf '%s' "$avg"
}
set_word_length() {
local length
length=${1:?}
# shellcheck disable=SC2183
__WORD_OFFSET_PATERN="$(printf '%*s' "$length" | tr ' ' '?')"
}
UNIQ_8_WORDS_COUNT=1000
DEFAULT_WORD_LENGTH=8
set_word_length "$DEFAULT_WORD_LENGTH"
randomword() {
local length
length=${1:-$DEFAULT_WORD_LENGTH}
LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c "$length"
}
WORDS=$(randomword $((8 * UNIQ_8_WORDS_COUNT)))
new_word() {
local prefix
prefix=${WORDS%"${WORDS#"$__WORD_OFFSET_PATERN"}"}
WORDS=${WORDS#"$__WORD_OFFSET_PATERN"}$prefix
printf '%s' "$prefix"
}
bench_set() {
local task depth count wordtime i start key d end
task=$1
depth=$2
count=$3
wordtime=$4
i=0
start=$(date +%s)
while [ "$i" -lt "$count" ]; do
key=$(new_word)
if [ "$depth" -gt 1 ]; then
d=1
while [ "$d" -lt "$depth" ]; do
key="$key.$(new_word)"
d=$((d + 1))
done
fi
plex_set 'MY_STRUCT' "$key" "$i"
i=$((i + 1))
done
end=$(date +%s)
elapsed "$task" "$((end - start))" "$count" "$(math "$wordtime*$depth")" >/dev/null
}
DEFAULT_TRIES=1000
ACCURATE_TRIES=10000
SUPPER_ACCURATE_TRIES=100000
WORD_CREATE_ACCURACY="$ACCURATE_TRIES"
BENCH_ACCURACY="$DEFAULT_TRIES"
count="$WORD_CREATE_ACCURACY"
set_word_length 8
i=0
start=$(date +%s)
while [ "$i" -lt "${count:?}" ]; do
new_word >/dev/null
i=$((i + 1))
done
end=$(date +%s)
wordtime=$(elapsed 'Word creation' "$((end - start))" "$count")
bench_set 'Set element with depth 1 length 8' 1 "$BENCH_ACCURACY" "$wordtime"
bench_set 'Set element with depth 2 length 8' 2 "$BENCH_ACCURACY" "$wordtime"
bench_set 'Set element with depth 3 length 8' 3 "$BENCH_ACCURACY" "$wordtime"
count="$WORD_CREATE_ACCURACY"
set_word_length 2
i=0
start=$(date +%s)
while [ "$i" -lt "${count:?}" ]; do
new_word >/dev/null
i=$((i + 1))
done
end=$(date +%s)
wordtime=$(elapsed 'Word creation' "$((end - start))" "$count")
bench_set 'Set element with depth 1 length 2' 1 "$BENCH_ACCURACY" "$wordtime"
bench_set 'Set element with depth 2 length 2' 2 "$BENCH_ACCURACY" "$wordtime"
bench_set 'Set element with depth 3 length 2' 3 "$BENCH_ACCURACY" "$wordtime"

19
package/hemar/parser/test/plex/jq_backend.sh
View File

@@ -1,19 +0,0 @@
. "${WORKSPACE:?}/src/plex/plex.sh"
init_plex yq-go
plex_set ZALUPA zalupa apulaz
log error "struct:\n$WHITE$(yq . "$PLEX_TEMP/ZALUPA.json")$NC"
plex_set ZALUPA kek.zalupa apulaz
log error "struct:\n$WHITE$(yq . "$PLEX_TEMP/ZALUPA.json")$NC"
plex_set ZALUPA zalupa apulaz
log error "struct:\n$WHITE$(yq . "$PLEX_TEMP/ZALUPA.json")$NC"
plex_val ZALUPA zalupa
plex_child ZALUPA kek
plex_fetch ZALUPA kek

128
package/hemar/parser/test/plex/jq_backend_time.sh
View File

@@ -1,128 +0,0 @@
#!/bin/dash
# shellcheck disable=SC1091
. "${WORKSPACE:?}/src/plex/plex.sh"
init_plex yq-go
math() {
awk "BEGIN {print $1}"
}
elapsed() {
local task time count decrease avg
task=$1
time=$2
count=$3
decrease=${4:-0}
avg=$(math "$time/$count-$decrease")
if [ "$time" -eq 0 ]; then
log info "\n[$WHITE${task}$NC]\ninstant\n"
else
log info "\n[$WHITE${task}$NC]\nelapsed $WHITE${avg}$NC seconds\n$WHITE$(math "1/$avg")$NC per second\n"
fi
printf '%s' "$avg"
}
set_word_length() {
local length
length=${1:?}
# shellcheck disable=SC2183
__WORD_OFFSET_PATERN="$(printf '%*s' "$length" | tr ' ' '?')"
}
UNIQ_8_WORDS_COUNT=1000
DEFAULT_WORD_LENGTH=8
set_word_length "$DEFAULT_WORD_LENGTH"
randomword() {
local length
length=${1:-$DEFAULT_WORD_LENGTH}
LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c "$length"
}
WORDS=$(randomword $((8 * UNIQ_8_WORDS_COUNT)))
WORDS=0123456789abcdefg
new_word() {
local prefix
# shellcheck disable=SC2295
prefix=${WORDS%"${WORDS#${__WORD_OFFSET_PATERN:?}}"}
# shellcheck disable=SC2295
WORDS=${WORDS#${__WORD_OFFSET_PATERN:?}}$prefix
printf '%s' "$prefix"
}
bench_set() {
local task depth count wordtime i start key d end
task=$1
depth=$2
count=$3
wordtime=$4
i=0
start=$(date +%s)
while [ "$i" -lt "$count" ]; do
key=$(new_word)
if [ "$depth" -gt 1 ]; then
d=1
while [ "$d" -lt "$depth" ]; do
key="$key.$(new_word)"
d=$((d + 1))
done
fi
set +e
plex_set 'MY_STRUCT' "$key" "$i"
error_code=$?
set -e
if [ $error_code != 0 ]; then
log error "key: $WHITE$key$NC, i: $WHITE$i$NC, struct: $WHITE$(jq . "$PLEX_TEMP/MY_STRUCT")$NC"
exit 1
fi
i=$((i + 1))
done
end=$(date +%s)
elapsed "$task" "$((end - start))" "$count" "$(math "$wordtime*$depth")" >/dev/null
}
DEFAULT_TRIES=1000
ACCURATE_TRIES=10000
SUPPER_ACCURATE_TRIES=100000
WORD_CREATE_ACCURACY="$SUPPER_ACCURATE_TRIES"
BENCH_ACCURACY="$DEFAULT_TRIES"
count="$WORD_CREATE_ACCURACY"
set_word_length 8
i=0
start=$(date +%s)
while [ "$i" -lt "${count:?}" ]; do
new_word >/dev/null
i=$((i + 1))
done
end=$(date +%s)
time=$((end - start))
log debug "word creation time: $time"
wordtime=$(elapsed 'Word creation' "$time" "$count")
bench_set 'Set element with depth 1 length 8' 1 "$BENCH_ACCURACY" "$wordtime"
bench_set 'Set element with depth 2 length 8' 2 "$BENCH_ACCURACY" "$wordtime"
bench_set 'Set element with depth 3 length 8' 3 "$BENCH_ACCURACY" "$wordtime"
log notice -
count="$WORD_CREATE_ACCURACY"
set_word_length 2
i=0
start=$(date +%s)
while [ "$i" -lt "${count:?}" ]; do
new_word >/dev/null
i=$((i + 1))
done
end=$(date +%s)
wordtime=$(elapsed 'Word creation' "$((end - start))" "$count")
bench_set 'Set element with depth 1 length 2' 1 "$BENCH_ACCURACY" "$wordtime"
bench_set 'Set element with depth 2 length 2' 2 "$BENCH_ACCURACY" "$wordtime"
bench_set 'Set element with depth 3 length 2' 3 "$BENCH_ACCURACY" "$wordtime"