diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cc3b90a..caa8679e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,14 +29,14 @@ - Options before a subcommand are read as the tool reads them (`docker -H h run`, `npm --prefix dir run build`, `yarn --cwd dir add`, `kubectl -o yaml get pods`, `gh -R o/r pr list`), and a tool's global options are told apart from its subcommands' (`kubectl logs -o` is not an option; terraform takes only `-chdir=DIR`, `-version` and `-help` before the subcommand). - The pitfall for a variable that another shell does not have (`bash -c`, `sudo`, `su -c`, `ssh`, `docker exec`) reads the string as that shell reads it, so `\$n` in `bash -c "echo \$n"` counts; `env -u x` removes `x`, and a shell that reads startup files (`bash -l`, an exported `BASH_ENV`) may have any name. The shell and env's `-i`, `-u` and `NAME=value` count in an `env -S` string too (`env -S'-i bash -c' '…'`). Before, an escaped `\$n` was missed, and a shell started from an `env -S` string was not known. - `test` reads no options, as bash: in `test 1 -eq 2 --version`, `-eq` and `--version` are words of the expression (bash says "`--version' unexpected"). Before, they were shown as options. -- A long option may be shortened to a unique prefix, as GNU tools allow, in `grep`, `sed`, GNU coreutils (`ls`, `cp`, `sort`, `tail`, `du`…), diffutils (`diff`, `cmp`), `gzip` and `tar`: `grep --invert` is `--invert-match`, `ls --almost` is `--almost-all`. These commands also know every long option of their `--help`. A prefix the tool finds ambiguous is shown as such. Before, these were "not in the dictionary". `grep` also knows `-G`, `--colour`, `--binary-files`, `--exclude-from` and the group separator options. `xargs --max-lines` and `mktemp --tmpdir` take a value only after `=`, as the tools do: in `xargs --max-lines echo`, `echo` is the command; before, it was taken for the value. Likewise `od -w f` reads `f` as input (before, as the width), and `install -t d a b` copies `a` and `b` into `d` (before, `-t` was unknown and `d` a file to copy). +- A long option may be shortened to a unique prefix, as GNU tools allow, in `grep`, `sed`, GNU coreutils (`ls`, `cp`, `sort`, `tail`, `du`…), diffutils (`diff`, `cmp`), `gzip` and `tar`: `grep --invert` is `--invert-match`, `ls --almost` is `--almost-all`. These commands also know every long option of their `--help`. A prefix the tool finds ambiguous is shown as such. Before, these were "not in the dictionary". `grep` also knows `-G`, `--colour`, `--binary-files`, `--exclude-from` and the group separator options. `xargs --max-lines` and `mktemp --tmpdir` take a value only after `=`, as the tools do: in `xargs --max-lines echo`, `echo` is the command; before, it was taken for the value. Likewise `od -w f` reads `f` as input (before, as the width), and `install -t d a b` copies `a` and `b` into `d` (before, `-t` was unknown and `d` a file to copy). `seq -w -2 2` reads `-2` as the first number (before, an unknown option), `tail +2` is `-n +2` (before, a file), `tr -C` is `-c`, and `cut -n` is known. In `awk 'NR>1' f -v` and `perl -ne '…' f -l`, the last word is a file, as the options end at the program (perl: at its first file); before, it was an option. Likewise `getopt -o a x -u` passes `-u` on to be parsed. perl's `-l0` and `-0777` take their digits, and `-0ne` is `-0 -n -e` (before, `ne` was the value of `-0`). `git log --max-count 2` is known. - In `kubectl`, `helm` and `gh`, a short option written with `=` takes what follows the `=` as its value, as these tools read it: `gh pr list -R=o/r` names the repository `o/r`. Before, the value was shown as `=o/r`. - `openssl` reads `--name` as `-name`, as OpenSSL does: in `openssl rand --hex 4`, `--hex` is `-hex`. Before, it was an unknown option. - `grep -E -F` (two different matchers of `-E`, `-F`, `-G`, `-P`) says that grep refuses the line with "conflicting matchers specified" and exit status 2, and so does `tar -c -x` ("You may not specify more than one…"). Such a line gets no example, and grep's pattern is not read as a regex. Before, it was explained, with its example, as a search with one of the matchers, and tar's line as one of the operations. - `bash -O extglob -c '…'` and `bash --posix` are read as bash's options. Before, `-O` was not known and `extglob` was taken for the script to run. - Quick fixes (the rewrites offered for dialect warnings) are offered in every language of the `bashExplainer.languages` setting, and follow a change of that setting, as the hover does. Before, they were offered in `shellscript` documents only. - `"$@"` inside single quotes is text, so the words after `bash -c 'f "$@"'` are no longer said to depend on how many words it gives. -- Examples and dialect checks read a command's options as the command reads them. `grep -e -v foo` searches for `-v`; its example said `-v` inverts the match. `sed -ie 's/a/b/' -e 's/c/d/' f` runs only `s/c/d/`, since `e` is the backup suffix of `-i`; the example ran both scripts. `sed -i.bak` now shows the file rewritten. The input of `grep -efoo data.txt | sort` and `awk '{ print }' v=1 | sort` is labelled as it is read (a file, then stdin), and a pipeline whose later stage reads a file instead of the pipe (`printf … | grep -e -v foo`) gets no whole-pipeline example. `tr --delete` is `-d`, and `find … -exec cmd {} '+'` runs the command once, quoted or not. The dialect check no longer takes `read -p -a v` for `read -a` (`-a` is the prompt), and finds `trap ERR`. +- Examples and dialect checks read a command's options as the command reads them. `grep -e -v foo` searches for `-v`; its example said `-v` inverts the match. `sed -ie 's/a/b/' -e 's/c/d/' f` runs only `s/c/d/`, since `e` is the backup suffix of `-i`; the example ran both scripts. `sed -i.bak` now shows the file rewritten. The input of `grep -efoo data.txt | sort` and `awk '{ print }' v=1 | sort` is labelled as it is read (a file, then stdin), and a pipeline whose later stage reads a file instead of the pipe (`printf … | grep -e -v foo`) gets no whole-pipeline example. `tr --delete` is `-d`, and `find … -exec cmd {} '+'` runs the command once, quoted or not. The dialect check no longer takes `read -p -a v` for `read -a` (`-a` is the prompt), and finds `trap ERR`. The examples of sort, uniq, cut, tr, head, tail, seq, basename, dirname, wc, xargs, sed, grep, awk, perl and `read` read the options as the explanation does, so a long prefix (`sort --rev`, `sed --expr`), `tail +2` and `tr -C` now get an example. `xargs --replace echo {}` and `xargs --max-lines echo` run `echo` (before, `echo` was taken for the value), and `-i`, `-l` are followed. `seq 1 -w 3`, which seq refuses, gets no example; before, it showed output. `cat -n`, `tac -s x` and `rev FILE` in a pipeline, and `cat -n f | sort`, get no example; before, the example passed the input on as if they had no option. The `git log` example shows as many commits as `--max-count 2` asks (before, 3). - Explanations match bash in more places: `x=1 ''` (a command name that is not found, so `x` is not set), the variables that `exec {fd}>file`, `wait -p var` and `coproc name` set, and an error in `$(( ))` (a division by 0) or an assignment to a readonly variable, which discards the rest of the line, also when it is in a function the line calls (before, only the rest of the command, or nothing). A name written in the option's own word (`printf -vx 2`, `read -rax`, `wait -n -px`) is the variable set (before, `x` kept its old value, and a condition on it could be called decided), and `wait -p x` unsets `x` before it sets it (before, `x` kept its old value when no job ended). `exec cmd` in `( )` or `$( )` says that it ends only the subshell (before, it said nothing of the subshell). `'exit' 3` with quotes has its exit status known, 3, as `exit 3` has (before, it was the status of an unknown command). `$(true)`, `$(:)` and `$(false)` count as empty only where the name runs the builtin, not after `true() { echo a; }`, an alias of it or `enable -n true` (before, `x=$(true)` was taken to set `x` to nothing there). Functions defined after `set -a +a` or `set foo -a` (where `-a` is an argument) are not exported, so conditions in them are decided (before, none were). In a function whose every call is in a subshell (`m() { cd /tmp; }; echo | m`), `cd` says "It may run in a subshell (where its function is called; what the script does not show may call it too)" when a sourced file, a trap's action or an alias may also call the function (before, "It runs in a subshell", although `. ./lib.sh` may call it in the shell itself); a word that names the function without calling it (`command m`, `declare -f m`, `type m`) no longer turns this into "may" (before, it did). `cmd &> >(tee log)` and `&>> >(…)` send both outputs to the process substitution, also after a block or a loop (`{ …; } &> >(tee log)`, `done &> >(tee log)`) and before the command name (`&> >(tee log) cmd`) (before, it was read as an argument of the command, and the redirection as missing its target; after a block or a loop it was a syntax error, and before the name it was the command). - `unset f` says it removes the function when there is no variable `f`, as bash does. A pipeline whose stage is a function or an alias of the script (`sort() { tac; }; … | sort`) no longer shows the output of the program of that name as an example. Before, the first said "Unset the variable" and the second showed the output of the program, which the function does not give. - The ⚠ notes on words and pipelines follow what bash does here. After `IFS=; set -f`, `cp $a b` no longer says that `$a` is split (the token already said it is not); `X=1 echo "\$X"` no longer warns that `$X` is expanded first (`\$X` is text); a variable set by `((count++))`, `printf -v` or `mapfile` at the end of a pipeline, or by a function called there, is now reported as lost after it, and `echo '$n'` no longer counts as a later read. An alias of `rm` is taken for `rm`. diff --git a/docs/design/refactor.md b/docs/design/refactor.md index 180dcf95..5196a29b 100644 --- a/docs/design/refactor.md +++ b/docs/design/refactor.md @@ -218,8 +218,8 @@ Verdicts are from the verification at 1f9b651. Line numbers are those of that co | S7 | Trust scan re-derives structure from text; two parsers per string | confirmed | 2.2 (eval, trap, indirection, slices), 3.2 (alias texts) | 2.2 part: PR #224; alias part planned, 3.2 | Indirection is read from the adapter's `!` token and name node (`indirect` in trust.ts), slices from the `:` operator token. This fixes indirections the regex missed, each of which may assign through a subscript: `${!1}`, `${!@}`, `${!*}`, `${!v@Q}`, `${!a[@]:-x}` (before: conditions after them decided). test/trust.test.ts "reads an indirection from the tree" (19 forms against bash; `${!#}`, `${!-}`, `${!a[@]}`, `${!v@}` stay trusted). The strings of `eval` and traps are parsed once, by the model's `Scripts` (`StringTrees`), and the scan reads those trees; `foreignParser` and the per-text dedupe are gone; the scan is made once per model (the trees are the model's), and its facts cover the strings' statements (`Trust.reads`), with what the whole script declares: test/trust.test.ts "reads the strings the walk runs from the walk's own trees" (`declare -A m; eval 'm[k]=1'`, bash). Left, taken over by 3.2: an alias's text is still parsed twice (the scan parses the value once per word through `StringTrees`; the walk parses it with the rest of each command it starts), and `Walker.facts` keeps a scan of that tree (`treeFacts`). sample.sh dump unchanged; explain time of a 400-line script of evals, traps and aliases: 1.4 s vs 1.5 s on main | | S8 | `aliasTrees` cache never evicts | confirmed (grows with distinct alias texts) | 1.5 | PR #208 | test/state.test.ts "the cache of alias texts" (300 alias texts, then a repeat: the values are right; the bound itself is not observable, so not tested) | | S9 | Stale doc (model.ts, layers diagram), c8 ignores, stray comment | corrected: words.ts:846–847 is a rethrow guard (needs a reason, stays); status.ts:363 is the unreachable one | 1.5 | PR #208 | docs and comments only; `enclosingBlock` has no unreachable return; coverage check | -| D1 | Embedded languages read options from raw words; `grep --invert` | confirmed; `grep --invert` part is D3's | 1.1 (readers), 3.5 (prefixes) | 1.1 part: PR #212; prefixes: PR #232 (grep, sed, tr); rest in 3.5 (awkOptions, yq `-o`) | cmdFlags, regexFlavor, sedScripts, jqVars, trContext, the perl letters and `-i`, the yq flags read the parse. Left for 3.5: awkOptions/awkInvocation and the yq `-o` format still read words; long prefixes (`grep --invert`, `sed --expr`, `tr --del`) are read since PR #232 (test/prefixes-real.test.ts) | -| D2 | Second option-parser family in src/examples; `grep -efoo data.txt \| sort` input label | confirmed (dump) | 1.1 (`readsFileOperand` bug), 3.5 (the parsers) | 1.1 part: PR #212 | `readsFileOperand` reads the operands of the parse (its `takesValue` table is gone); later stages that read a file give no pipeline example. "readsFileOperand agrees with the tools…" and "the pipeline example of a line agrees with the tools" (test/examples-pipeline.test.ts). parseOptions, sedArgs, perlOptions stay for 3.5 | +| D1 | Embedded languages read options from raw words; `grep --invert` | confirmed; `grep --invert` part is D3's | 1.1 (readers), 3.5 (prefixes) | 1.1 part: PR #212; prefixes: PR #232 (grep, sed, tr); the rest: PR #248 | cmdFlags, regexFlavor, sedScripts, jqVars, trContext, the perl letters and `-i`, the yq flags read the parse. Since PR #248, awkInvocation and the yq `-o` format read the parse too (awk reads options only before the program, as gawk: examples-awk, options-fixes "awk" rows); long prefixes (`grep --invert`, `sed --expr`, `tr --del`) are read since PR #232 (test/prefixes-real.test.ts) | +| D2 | Second option-parser family in src/examples; `grep -efoo data.txt \| sort` input label | confirmed (dump) | 1.1 (`readsFileOperand` bug), 3.5 (the parsers) | 1.1 part: PR #212; the parsers: PR #248 | `readsFileOperand` reads the operands of the parse (its `takesValue` table is gone); later stages that read a file give no pipeline example. "readsFileOperand agrees with the tools…" and "the pipeline example of a line agrees with the tools" (test/examples-pipeline.test.ts). PR #248: `parseOptions` (textfilters.ts) builds its result from `interpret()` with the command's dictionary spec (`parseWords`); its tables only say which options a simulator follows, keyed by any name of the spec's option, and an unknown, ambiguous or unsupported option, or a value not literal, gives no example. Its callers (sort, uniq, cut, tr, head, tail, seq, basename, dirname, wc, grep, `read`, xargs), `sedArgs`, `perlOptions`, `awkInvocation`, xargs's `-0`/`-d` check and the yq `-o` format read the parse; `sortsUnique` replaces a regex. Example tables unchanged, plus rows for what is now followed (`sort --rev`, `wc --li`, `cut --comp`, `tr -C`, `cut -n`, `head -q`, `uniq --all-repeated=none`, xargs `--replace`/`-i`/`-l`/`--max-lines`) and declined as the tools refuse (`grep -E -F`, `seq 1 -w 3`), in examples-textfilters, examples-pipeline, examples-runtime. Spec fixes found, against the tools (test/options-fixes.test.ts): seq reads options first and a negative number is an operand (`seq -w -2 2`, `seq 10 -3 1`; `endsOptions` is now tested before the word is read as an option), `tail +2`, awk and perl read options only before the program, `tr -C`, `cut -n`. The cat, tac and rev runners pass the input on only without options or files (`cat -n` numbers the lines; examples-pipeline "declines cat, tac and rev…", against the tools), and a first `cat FILE` stage reads the parse (`cat -n f | sort`: no example). Review: the getopt example (src/examples/commands.ts) reads the parse, and getopt reads its own options only before the first operand (`getopt -o a x -u`: `-u` is parsed, options-fixes, against util-linux getopt); `gitLogExample` reads the count, format and date from the parse (`--max-count 2`, `-n2`, `-2`, the last winning; examples-gitlog, against git); yq v4 uses the `cobra` preset, so `-o=json` is `json` as pflag reads it (yq.test "yq -o as pflag reads it"; mikefarah/yq is not installed here, the row against it runs where it is); perl's `-l` and `-0` take octal digits attached and other letters after them are switches (`ValueSpec.attachedMatch`: `-l0`, `-0777`, `-lne`, `-0ne`; options-fixes, against perl; `-l0ne` is not modelled). sample.sh dump unchanged. Left: `sed -i` on stdin is simulated (GNU sed fails) | | D3 | `longPrefixes` opt-in; `grep --invert` unknown; five flags describe a family | corrected: off by default is decision 3 of semantics.md; the gap is that getopt_long tools like grep lack it | 3.5 | presets: PR #230; GNU entries of commands.ts: PR #232; the other GNU entries, grep's matchers, OpenSSL's `--name`: PR #247; left: util-linux, procps, systemd, binutils and other getopt_long tools, Go flag (later step) | `parser` presets (src/argv/parsers.ts: `gnu`, `getopt_long`, `single-dash`, `letters`, `cobra`, `iproute2`, `jq`, `git`, `dpkg`) and `optionsFirst` replace `style`, `longPrefixes`, `longEquals`, `shortClusters`, `shortEquals`, `cobra`. test/parsers-real.test.ts compares each preset with a real parser of its family (getopt(1) on 17 lines for gnu and getopt_long, with and without `+`; bash echo; openssl; gh; ip; jq; git; dpkg-query) and fails when a preset has no rows. A script compared the rules of all 1517 specs before and after (with `optionsFirst`): the same, except cobra and builtin `test`. Fixes found: cobra reads `-R=foo` as `foo`, as pflag does (parsers-real "cobra" rows); builtin `test` (single-dash, which no longer cancels the builtins' `optionsFirst`) shows no word as an option, as bash reads none (parsers-real "test reads no options, as bash", 6 rows against bash); CHANGELOG for both. Under CI a missing tool fails the preset's rows; elsewhere the last row lists the presets not compared. sample.sh dump unchanged. Left for 3.5: turning `getopt_long` on for the GNU entries (their --help lists checked), OpenSSL and Go flag accept `--name` as `-name` (single-dash does not). GNU entries (PR #232): grep, sed, cut, head, tr, cat, tee, paste, seq, mktemp, basename, dirname use `getopt_long`, with `--help` and `--version` (`GNU_INFO`) and the long options their `--help` lists (grep: `-G`, `--colour`, `--binary-files`, `--exclude-from`, the group separators, `--fixed-regexp`). test/prefixes-real.test.ts checks every entry with `getopt_long`: each of its long options is one the tool has, and the shortest prefix unique among the entry's and the tool's names is read as the tool reads it (glibc's message); the tool's options the entry lacks are only listed, so a newer release does not fail; it fails when an entry with `getopt_long` is not in its list. Fixes it found: `xargs --max-lines` and `mktemp --tmpdir` take a value only attached (optional_argument), so the next word is the command or the template. "lines with long options" rows: `grep --invert` (D1), the ambiguous `grep --in`, `sed --s`, `cut --c`, `head --v` (same candidates as the tool), `grep --col=never`, `xargs --max-lines echo`, `mktemp --tmpdir x.XXXX`, against the tools. sample.sh dump unchanged. Left for 3.5: the GNU entries of the other files and those of commands.ts whose `--help` has options the entry lacks (tail, wc, uniq, sort, ls, cp, mv, rm, du, diff, tar…). Other GNU entries (PR #247): coreutils in every dict file (ls, sort, uniq, tail, wc, cp, mv, rm, du, df, install, the checksums, pr, numfmt… about 90 entries; not `dd` and `expr`, which read their own words), diffutils (diff, cmp, sdiff, diff3), gzip, gunzip, zcat and GNU tar use `getopt_long` with every long option of their `--help` (and the hidden ones glibc counts for ambiguity: `diff --binary`, gzip `--ascii`, `cksum --binary`, tar `--usage`…), each in test/prefixes-real.test.ts's list; stty checks its words before getopt_long, so its prefixes are checked with ours and a row runs `stty --sa`. Value fixes it found: `od -w`/`--width` and `split --numeric-suffixes` take an optional value, `install -t` was missing (`install -t d a b`: two files to copy, test/positional-count.test.ts), the long forms of `ls -F`, `cp -u`, `tail -f`, `diff -u`/`-c`, `shred -u` take an optional value; tar's `-k` and `--wildcards` texts. `grep -E -F`: grep's matchers select modes and `CommandSpec.conflictingModes` adds grep's refusal to the second one; the parse is `refused`, so the line gets no example and its pattern no regex reading (prefixes-real "grep -E -F a: refused, as grep does", 9 rows against GNU grep, with the example and the pattern; `egrep -F` is not detected: the alias's implied `-E` is not in the parse). tar's operations use it too (`tar -c -x`, `tar cxf`: "You may not specify more than one", 6 rows against GNU tar). Preset `openssl` (single-dash, and `--name` is `-name`; parsers-real `openssl rand --hex 4`), single-dash now compared with gcc (`--dumpversion` refused). Go's flag package also reads `--name`, but no Go tool is installed here to compare: go, terraform, shfmt stay on `single-dash`. test/oldstyle-real.test.ts reads tar's `--help` lines whose name fills its column. prefixes-real tolerates other releases of the tools: an option of the entry the tool here does not know (and its `--help` does not list), and a prefix made ambiguous by a hidden option neither lists, are only printed; its comment (RELEASES) says what to do when the CI runner moves. sample.sh dump unchanged | | D4 | Host identity is its English label | confirmed | 1.4 | PR #211 | `ShellRegion.host` (`HostId`: kind, ci) read by hosts.ts, embedded/yq.ts and analyzer.ts; test/hosts.test.ts "a region says what it is part of, apart from its label" | | D5 | Seven+ program-name → Shell mappings that disagree; dash/ksh read as sh | confirmed (eight in hosts.ts; ksh warned "fails in dash") | 1.4 | PR #211 | `shellFromName` exported from dialect.ts and used at every site in hosts.ts; test/hosts.test.ts "one mapping from a program name to a shell" (dash against the real dash; ksh is not installed, so ksh is checked against the mapping table and the `[[ ]]` support list); Dockerfile exec-form `RUN` gate and env.ts's `shell-string` slot use it too (test/hosts.test.ts "posh, busybox and lksh are shells wherever a shell is named", test/env.test.ts "the shell a program name stands for"). Rest in 3.4: YAML `shell:` accepts only bash/sh and Markdown fences only zsh/ksh/dash, because widening them changes which CI lines are explained; `lookupCommand`'s list is D6 | diff --git a/src/analyzer.ts b/src/analyzer.ts index 304f31f6..91a3171f 100644 --- a/src/analyzer.ts +++ b/src/analyzer.ts @@ -2160,7 +2160,7 @@ export class Analyzer { break; case 'gitformat': case 'gitdate': - this.pushExample(gitLogExample(lang, text, this.cx.argCtx!.args.map((x) => literalContent(x)?.text ?? ''), this.lang)); + this.pushExample(gitLogExample(lang, text, this.cx.argCtx!.parse, this.cx.argCtx!.nodes.map((x) => literalContent(x)?.text), this.lang)); break; case 'glob': { // Unlike the shell, find matches a leading dot with * too; -path also matches / with *. @@ -2333,10 +2333,11 @@ export class Analyzer { /** Input and output of a yq expression, in the YAML each yq reads and prints. */ private yqExample(filter: string, style: YamlStyle): void { - const words = this.cx.argCtx!.args.map((a) => literalContent(a)?.text ?? ''); // mikefarah/yq: only expressions made of operations that behave as in jq (checked against yq). if (style === 'mikefarah' && !yqSameAsJq(filter)) return; - const format = words.map((w, i) => (/^(-o|--output-format)$/.test(w) ? words[i + 1] : /^-o=?(.+)$/.exec(w)?.[1] ?? /^--output-format=(.+)$/.exec(w)?.[1])).find(Boolean); + // (The last `-o` wins, as pflag reads it; a format that is not plain text is none the example follows.) + const given = this.optionValues('-o', '--output-format'); + const format = given.length ? given[given.length - 1].text ?? '?' : undefined; const json = style === 'mikefarah' ? format === 'json' || format === 'j' : !this.given('-y', '-Y'); if (style === 'mikefarah' && format && !json && format !== 'yaml' && format !== 'y') return; const raw = style === 'mikefarah' ? !json : this.given('-r'); diff --git a/src/argv/interpret.ts b/src/argv/interpret.ts index e549ae61..418cf380 100644 --- a/src/argv/interpret.ts +++ b/src/argv/interpret.ts @@ -186,7 +186,9 @@ class Reader { const used = this.oldStyle(st, i); if (used >= 0) { i = used; continue; } } - if (!endOpts) { + // (A word `endsOptions` matches is an operand even when it looks like an option: seq's `-2`.) + const ends = !!spec.endsOptions && value !== undefined && spec.endsOptions.test(value); + if (!endOpts && !ends) { const used = this.option(st, i); if (used >= 0) { const split = this.splice(st, i, used); @@ -210,7 +212,7 @@ class Reader { break; } // Options end at the first operand (`optionsFirst`), or where a command starts. - if (rules.optionsFirst || this.startsCommand(st, [...operands, i]) || (spec.endsOptions && value !== undefined && spec.endsOptions.test(value))) endOpts = true; + if (rules.optionsFirst || this.startsCommand(st, [...operands, i]) || ends) endOpts = true; operands.push(i); } parse.mode = chooseMode(spec, given); @@ -479,6 +481,8 @@ class Reader { const first = values[0]; // `set -euo pipefail`: the value is the next word, and the letters after it still count. if (first.how === 'next-only') { last = this.separateValues(st, opt, as, last, 0, role); continue; } + // perl's `-lne`: the rest is not digits, so `-l` has no value and `n`, `e` are options. + if (first.attachedMatch && !open && !first.attachedMatch.test(text.slice(k + 1))) continue; if (k + 1 < text.length || open) { // `dpkg-query -f=FORMAT`: the value starts after the `=`. const at = st.rules.shortEquals && text[k + 1] === '=' ? k + 2 : k + 1; diff --git a/src/argv/spec.ts b/src/argv/spec.ts index 015f5f23..36d3e2ad 100644 --- a/src/argv/spec.ts +++ b/src/argv/spec.ts @@ -54,9 +54,10 @@ export interface CommandSpec { */ subcommandPrefixes?: boolean | 'first'; /** - * A word that matches is an operand and ends the options and the old-style letters: every word after it is an - * operand too (procps `ps 1 u`: a bare number starts a list of process IDs, which ends the line). (procps `ps` - * only: its own parser, which no preset describes.) + * A word that matches is an operand, even when it starts with a dash, and ends the options and the old-style + * letters: every word after it is an operand too (procps `ps 1 u`: a bare number starts a list of process IDs, + * which ends the line; GNU `seq -2 2`: a negative number is the first operand). (procps `ps` and GNU `seq`: each + * checks the word before its parser reads it, which no preset describes.) */ endsOptions?: RegExp; /** @@ -166,6 +167,12 @@ export interface ValueSpec { optional?: boolean; /** With `optional`, the next word is the value when it matches (`journalctl -b -1`). */ next?: RegExp; + /** + * With `optional`, the rest of a cluster is the value only when it matches; otherwise the letters after the option + * are options too (perl's `-l0` and `-0777` take octal digits, `-lne` is `-l -n -e`). (perl only: its switches + * take digits attached and go on with letters; a value of digits followed by letters, `-l0ne`, is not modelled.) + */ + attachedMatch?: RegExp; /** `variable`: a variable name, which the builtin sets as `sets` says; `function`: a function it runs later (`mapfile -C`, `complete -F`). */ type?: ValueType; sets?: VariableSet; diff --git a/src/dict/commands.ts b/src/dict/commands.ts index 7faf12bd..595067ad 100644 --- a/src/dict/commands.ts +++ b/src/dict/commands.ts @@ -2,7 +2,7 @@ import { shellOptions, shoptOptions } from './builtins'; import { ops } from './ops'; import { distro } from './distro'; import { system } from './system'; -import type { CommandSpec, OperandAlt, OperandSpec } from '../argv/spec'; +import type { CommandSpec, OperandAlt, OperandSpec, OptionSpec, PatternOption } from '../argv/spec'; import { tx } from '../i18n'; import { basic } from './basic'; import { storage } from './storage'; @@ -212,6 +212,15 @@ const LINE_COUNT = countOption( pos('展開した数の行数（`-n 数` の古い書き方）', 'as many lines as the expansion gives (old way to write `-n N`)'), ); +/** A perl switch that takes digits attached (`-l0`, `-0777`); other letters after it are switches (`-lne`). */ +const perlDigits = (opt: OptSpec, digits: RegExp): OptionSpec => ({ ...optionOf(opt), value: { name: 'VALUE', how: 'attached-only', optional: true, attachedMatch: digits } }); + +/** `tail +2`: the old way to write `-n +2` (GNU tail takes it as the first word, before at most one file). */ +const FROM_LINE: PatternOption = { + pattern: /^\+(\d+)$/, + text: ([, n]) => pos(`${n} 行目から最後まで（\`-n +${n}\` の古い書き方）`, `from line ${n} to the end (old way to write \`-n +${n}\`)`), +}; + /** awk's files, or `name=value` assignments between them. */ const AWK_FILES: OperandSpec = { name: 'ARG2', text: files, repeat: '*', @@ -330,6 +339,8 @@ export const core: Record = { { when: { option: ['-f'] }, pattern: [AWK_FILES] }, { pattern: [{ name: 'ARG1', text: pos('awk プログラム（`パターン { アクション }`）', 'awk program (`pattern { action }`)'), repeat: '?', lang: 'awk' }, AWK_FILES] }, ], + // (The options end at the program: gawk's getopt string starts with `+`; POSIX awk too.) + optionsFirst: true, }), find: findSpec(cmd('条件に一致するファイルをディレクトリ階層から探す', 'Search a directory tree for files matching conditions', [ ['-name', 'ファイル名がグロブパターンに一致する', 'file name matches the glob', 'arg lang:glob'], @@ -414,6 +425,7 @@ export const core: Record = { ['-f|--fields', '取り出すフィールド番号（例: `1,3` や `2-`）', 'fields to select (e.g. `1,3`, `2-`)', 'arg lang:cutfields'], ['-c|--characters', '取り出す文字位置', 'character positions to select', 'arg lang:cutchars'], ['-b|--bytes', '取り出すバイト位置', 'byte positions to select', 'arg lang:cutchars'], + ['-n', '何もしない（互換のため受け付ける）', 'ignored (accepted for compatibility)'], ...GNU_INFO, ], { example: 'cut', parser: 'getopt_long', positional: [files] }), sort: cmd('行を並べ替える', 'Sort lines', [ @@ -455,11 +467,15 @@ export const core: Record = { ['-s|--shell', 'クォートの仕方を指定したシェルに合わせる（sh, bash, csh, tcsh）', 'quote for the given shell (sh, bash, csh, tcsh)', 'arg'], ['-u|--unquoted', '出力をクォートしない（空白を含む引数が壊れる）', 'do not quote the output (breaks arguments with spaces)'], ['-T|--test', '拡張版の getopt か確認する（拡張版なら終了ステータス 4）', 'test for the enhanced getopt (exits 4 if so)'], - ], { example: 'getopt', positional: [pos('（旧形式）受け付けるオプション文字', '(old form) the option letters'), pos('解析する引数（通常は `"$@"`）', 'the arguments to parse (usually `"$@"`)')], positionalLang: ['getoptshort'] }), + ], { + example: 'getopt', + // util-linux getopt reads its own options up to the first operand (`+` in its getopt string): the words after it are the arguments it parses. + optionsFirst: true, + positional: [pos('（旧形式）受け付けるオプション文字', '(old form) the option letters'), pos('解析する引数（通常は `"$@"`）', 'the arguments to parse (usually `"$@"`)')], positionalLang: ['getoptshort'] }), tr: cmd('文字を置換・削除する（標準入力のみ）', 'Translate or delete characters (stdin only)', [ ['-d|--delete', '指定した文字を削除する', 'delete the characters'], ['-s|--squeeze-repeats', '連続する同じ文字を1つにまとめる', 'squeeze repeated characters'], - ['-c|--complement', '指定した文字以外を対象にする', 'use the complement of the set'], + ['-c|-C|--complement', '指定した文字以外を対象にする', 'use the complement of the set'], ...GNU_INFO, ], { example: 'tr', parser: 'getopt_long', positional: [pos('変換元の文字集合（例: `a-z`）', 'source character set (e.g. `a-z`)'), pos('変換先の文字集合（例: `A-Z`）', 'target character set (e.g. `A-Z`)')], positionalLang: ['trset', 'trset'], operands: [byCount(1, [pos('削除または圧縮する文字集合（`-d`、`-s`）', 'character set to delete or squeeze (`-d`, `-s`)')], { langs: ['trset', 'trset'] })] }), head: cmd('ファイルの先頭部分を表示する（既定は10行）', 'Print the first part of files (default 10 lines)', [ @@ -475,7 +491,7 @@ export const core: Record = { ['-F', 'ファイルが作り直されても追い続ける', 'follow by name, retrying'], ['--max-unchanged-stats', '`-f` で変化がなくても、ファイルが入れ替わっていないか確かめるまでの回数', 'with `-f`, checks without change before testing whether the file was replaced', 'arg'], ...GNU_INFO, - ], { example: 'tail', parser: 'getopt_long', positional: [files], patternOptions: [LINE_COUNT] }), + ], { example: 'tail', parser: 'getopt_long', positional: [files], patternOptions: [LINE_COUNT, FROM_LINE] }), wc: cmd('行数・単語数・バイト数を数える', 'Count lines, words and bytes', [ ['-l|--lines', '行数を数える', 'count lines'], ['-w|--words', '単語数を数える', 'count words'], @@ -1211,7 +1227,8 @@ export const core: Record = { ['-f|--format', 'printf 形式の書式（`%03g`…）', 'printf-style format (`%03g`…)', 'arg'], ...GNU_INFO, ], { - parser: 'getopt_long', + // getopt_long with `+`, and a word that looks like a negative number (`-2`, `-.5`) is the first operand. + parser: 'getopt_long', optionsFirst: true, endsOptions: /^-[.\d]/, example: 'seq', positional: [pos('開始値・増分・終了値', 'first, increment or last')], operands: [ @@ -1411,14 +1428,17 @@ export const core: Record = { ['-n', '入力の各行についてコードを実行する（自動では出力しない）', 'run the code for each input line (no automatic printing)'], ['-p', '入力の各行についてコードを実行し、`$_` を出力する（sed 風）', 'run the code for each line and print `$_` (sed-like)'], ['-i', 'ファイルを直接書き換える（`-i.bak` でバックアップを作る）', 'edit files in place (`-i.bak` keeps a backup)', 'attached'], - ['-l', '各行末の改行を取り除き、print で改行を付ける', 'chomp input lines and add a newline to print'], + perlDigits(['-l', '各行末の改行を取り除き、print で改行を付ける（`-l0` のように8進数を付けると、改行の代わりにその文字を付ける）', 'chomp input lines and add a newline to print (with octal digits, as in `-l0`, that character instead)'], /^[0-7]+$/), ['-a', '各行を空白で分割して `@F` に入れる（-n か -p と使う）', 'split each line into `@F` (with -n or -p)'], ['-F', '`-a` の区切り（例: `-F:`、`-F,`）', 'separator for `-a` (e.g. `-F:`)', 'attached'], - ['-0', '入力の区切り（`-0777` でファイル全体を1つとして読む）', 'input record separator (`-0777` reads whole files)', 'attached'], + perlDigits(['-0', '入力の区切り（`-0777` でファイル全体を1つとして読む）', 'input record separator (`-0777` reads whole files)'], /^(?:[0-7]+|x[0-9a-fA-F]+)$/), ['-M', 'モジュールを読み込む（例: `-MJSON::PP`）', 'load a module (e.g. `-MJSON::PP`)', 'attached'], ['-w', '警告を表示する', 'enable warnings'], ['-c', '構文チェックだけを行う', 'check syntax only'], - ], { example: 'perl', positional: [pos('実行するスクリプトファイル', 'script file to run'), files] }), + ], { + example: 'perl', + // perl reads switches up to the first word that is not one: the script, or with `-e` the first file. + optionsFirst: true, positional: [pos('実行するスクリプトファイル', 'script file to run'), files] }), python3: cmd('Python 3 インタプリタを実行する', 'Run the Python 3 interpreter', [ ['-c', '続く文字列を Python コードとして実行', 'run the string as Python code', 'arg'], ['-m', 'モジュールをスクリプトとして実行', 'run a module as a script', 'arg'], @@ -1486,7 +1506,7 @@ export const core: Record = { }), status: cmd('作業ツリーの状態を表示する', 'Show working tree status', [['-s|--short', '短い形式', 'short format'], ['--porcelain', 'スクリプト向けの安定した形式', 'stable machine-readable format']]), log: cmd('コミット履歴を表示する', 'Show commit logs', [ - ['--oneline', '1コミット1行で表示', 'one line per commit'], ['-n', '表示する件数', 'number of commits', 'arg'], ['--graph', 'ブランチをグラフ表示', 'draw a graph'], + ['--oneline', '1コミット1行で表示', 'one line per commit'], ['-n|--max-count', '表示する件数', 'number of commits', 'arg'], ['--graph', 'ブランチをグラフ表示', 'draw a graph'], ['--pretty|--format', '各コミットの表示形式', 'how each commit is shown', 'arg lang:gitformat'], ['--date', '日時の表示形式', 'how dates are shown', 'arg lang:gitdate'], ['--since|--after', '指定した日時以降のコミットだけ', 'commits after the given date', 'arg'], diff --git a/src/dict/yq.ts b/src/dict/yq.ts index 48cd5945..79b361ca 100644 --- a/src/dict/yq.ts +++ b/src/dict/yq.ts @@ -32,9 +32,10 @@ const V4_OPTIONS: OptSpec[] = [ const V4_EXPR = { positional: [pos('yq の式（jq に似た記法）', 'yq expression (jq-like)'), pos('入力ファイル（省略すると標準入力）', 'input file (stdin if omitted)')], positionalLang: ['yq4' as const] }; -/** mikefarah/yq v4: `yq [eval] EXPR FILE…`. */ +/** mikefarah/yq v4: `yq [eval] EXPR FILE…`; a Go program with cobra and pflag (`-o=json` gives `-o` the value `json`). */ const mikefarah4: CommandSpec = cmd('YAML（JSON・XML・TOML なども）を jq に似た式で抽出・変換する（mikefarah/yq v4）', 'Query and edit YAML (also JSON, XML, TOML…) with jq-like expressions (mikefarah/yq v4)', V4_OPTIONS, { ...V4_EXPR, + parser: 'cobra', subcommands: { e: cmd('式を各ドキュメントに順に適用する（`eval`）', 'apply the expression to each document (`eval`)', V4_OPTIONS, V4_EXPR), eval: cmd('式を各ドキュメントに順に適用する', 'apply the expression to each document', V4_OPTIONS, V4_EXPR), diff --git a/src/examples/awk.ts b/src/examples/awk.ts index f1b9e667..e26fd5c5 100644 --- a/src/examples/awk.ts +++ b/src/examples/awk.ts @@ -1,6 +1,8 @@ import { lex, Lex } from '../embedded/awk'; import { JsRegex, regexExamples, toJsRegex } from './regex'; import { Unsupported as RunnerUnsupported } from './unsupported'; +import { parseWords } from './textfilters'; +import { has } from '../argv/query'; /** Raised for awk features the example runner does not implement; no example is shown then. */ class Unsupported extends RunnerUnsupported {} @@ -1070,41 +1072,43 @@ export interface AwkInvocation extends AwkRunOptions { unsupported?: string; } -/** Read the options before the program and the `name=value` operands after it. A word that is not literal is undefined. */ +/** + * Read the options before the program and the `name=value` operands after it, with awk's dictionary spec (as the + * explanation reads them). A word that is not literal is undefined. + */ export function awkInvocation(words: (string | undefined)[]): AwkInvocation { const r: AwkInvocation = { vars: {}, assigns: {} }; const bad = (why: string) => { r.unsupported ??= why; }; - let i = 0; - let fromFile = false; - for (; i < words.length; i++) { - const w = words[i]; - if (w === undefined) { bad('non-literal'); continue; } - if (w === '--') { i++; break; } - if (w === '-F' || /^-F./.test(w)) { - const v = w === '-F' ? words[++i] : w.slice(2); + // The value of `-F` or `-v` (undefined: missing or not literal). + const take = (flag: string, v: string | undefined): void => { + if (flag === '-F') { // `-F'\t'` is a tab; `-Ft` is the letter t in gawk and mawk alike. const fs = v === undefined ? undefined : awkUnescape(v, true); if (fs === undefined) bad('-F'); else r.fs = fs; - continue; - } - if (w === '-v' || /^-v./.test(w)) { - const v = w === '-v' ? words[++i] : w.slice(2); + } else if (flag === '-v') { const eq = v?.indexOf('=') ?? -1; const value = v === undefined || eq < 1 ? undefined : awkUnescape(v.slice(eq + 1)); if (v === undefined || value === undefined) bad('-v'); else r.vars[v.slice(0, eq)] = value; - continue; } - if (w === '-f') { fromFile = true; i++; continue; } - if (w.startsWith('-')) { bad(`awk ${w}`); continue; } - break; + }; + const parse = parseWords('awk', words); + const rest: (string | undefined)[] = []; + // In the order of the words, so that the first reason is the one given. + for (const { word, role } of parse.roles) { + for (const x of role.role === 'cluster' ? role.letters : [role]) { + // Options awk has more of (`--posix`, `-W …`) do not change what the example shows; the caller decides. + if (x.role === 'unknown-option') bad(`awk ${words[word]}`); + else if (x.role === 'option' && (x.value || x.missing)) take(x.flag, x.value && words[word]!.slice(x.value.attached)); + else if (x.role === 'value') take(x.of, words[word]); + else if (x.role === 'operand') { if (words[word] === undefined) bad('non-literal'); rest.push(words[word]); } + } } - if (!fromFile) r.program = words[i++]; + if (!has(parse, '-f')) r.program = rest.shift(); let seenFile = false; - for (; i < words.length; i++) { - const w = words[i]; - const m = w === undefined ? null : /^([A-Za-z_]\w*)=([^]*)$/.exec(w); - if (w === undefined) bad('non-literal'); - else if (!m) seenFile = true; + for (const w of rest) { + if (w === undefined) continue; + const m = /^([A-Za-z_]\w*)=([^]*)$/.exec(w); + if (!m) seenFile = true; else if (seenFile) bad('assignment after a file'); else { const value = awkUnescape(m[2]); diff --git a/src/examples/commands.ts b/src/examples/commands.ts index ce98a3eb..fda3e1bb 100644 --- a/src/examples/commands.ts +++ b/src/examples/commands.ts @@ -13,7 +13,9 @@ import { perlExample, perlOptions } from './perlrun'; import { Interactive, rerun, RerunSpec } from './rerun'; import { readSampleLine, runRead, xargsExample as runXargs } from './runtime'; import { bashPrintf, ShellWordsUnsupported } from './shellwords'; -import { commandExample, parseOptions, Unsupported as TextFiltersUnsupported } from './textfilters'; +import { commandExample, parseOptions, parseWords, shortKeys, Unsupported as TextFiltersUnsupported } from './textfilters'; +import type { ArgvParse } from '../argv/interpret'; +import { has, values } from '../argv/query'; /** A word of the command line: its text when it is known (`literalContent`), and as written. */ export interface ExampleWord { value: string | undefined; text: string } @@ -105,7 +107,7 @@ function xargsExample(words: ExampleWord[], lang: Lang): Example | undefined { { label: s('実行されるコマンド', 'Commands that run'), lines: ex.commands, empty: s('（実行されない）', '(none)') }, ], // Input split on NUL or another delimiter is not one item per line, so it is not editable. - ...(words.some((w) => /^(-0|--null|-d|--delimiter)/.test(w.text)) ? {} : { interactive: interactive({ kind: 'xargs', words: values }, input, lang, s('入力（標準入力）', 'Input (stdin)')) }), + ...(has(parseWords('xargs', values), '-0', '-d') ? {} : { interactive: interactive({ kind: 'xargs', words: values }, input, lang, s('入力（標準入力）', 'Input (stdin)')) }), }; } @@ -113,42 +115,25 @@ function xargsExample(words: ExampleWord[], lang: Lang): Example | undefined { function getoptExample(words: (string | undefined)[], lang: Lang): Example | undefined { const s = pick(lang); const spec: GetoptSpec = { short: '', long: [] }; - let i = 0; - const value = (w: string, short: string, long: string[]): string | undefined => { - if (w === short || long.includes(w)) return words[++i]; - if (w.startsWith(short) && w.length > 2) return w.slice(2); - const eq = long.find((l) => w.startsWith(`${l}=`)); - return eq ? w.slice(eq.length + 1) : undefined; - }; - let shortGiven = false; - for (; i < words.length; i++) { - const w = words[i]; - if (w === undefined) return undefined; - if (w === '--') break; - if (!w.startsWith('-')) { - // The old form: `getopt OPTSTRING ARGS…`. - if (!shortGiven) { - spec.short = w; - spec.legacy = true; - } - break; - } - if (/^-(o|-options)/.test(w)) { - const v = value(w, '-o', ['--options']); - if (v === undefined) return undefined; - spec.short = v; - shortGiven = true; - } else if (/^-(l|-long)/.test(w)) { - const v = value(w, '-l', ['--longoptions', '--long']); - if (v === undefined) return undefined; - spec.long.push(...v.split(',')); - } else if (/^-(n|s|-name|-shell)/.test(w)) { - if (!w.includes('=') && w.length === 2 || ['--name', '--shell'].includes(w)) i++; - } else if (w === '-a' || w === '--alternative') { - spec.alternative = true; - } else if (/^-(Q|T|-quiet-output|-test)$/.test(w)) { - return undefined; - } + // getopt's own options, read with its spec; `-Q` and `-T` print nothing to show, and a value not known gives none. + let p; + try { + p = parseOptions('getopt', words, shortKeys('olaqu', { '-n': '_', '-s': '_' })); + } catch (e) { + if (e instanceof TextFiltersUnsupported) return undefined; + /* c8 ignore next */ + throw e; + } + for (const [key, v] of p.opts) { + if (key === 'o') spec.short = v; + else if (key === 'l') spec.long.push(...v.split(',')); + else if (key === 'a') spec.alternative = true; + } + // The old form: `getopt OPTSTRING ARGS…`, when no `-o` is given. + if (!p.opts.some(([key]) => key === 'o') && p.operands.length) { + if (p.operands[0] === undefined) return undefined; + spec.short = p.operands[0]; + spec.legacy = true; } if (!spec.short && !spec.long.length) return undefined; try { @@ -185,7 +170,7 @@ export function readExample(words: (string | undefined)[], ifs: string | undefin if (ifs === undefined || words.some((w) => w === undefined)) return undefined; let p; try { - p = parseOptions(words, { flags: 'rse', values: 'aptu', long: {} }); + p = parseOptions('read', words, shortKeys('rseaptu')); } catch (e) { // An option the example does not model (`-n 3`): no example. Anything else is a bug, which shows. if (e instanceof TextFiltersUnsupported) return undefined; @@ -230,16 +215,25 @@ export function dateExample(format: string, lang: Lang): Example | undefined { }; } -/** `git log` output for sample commits, for a `--pretty`/`--format` or `--date` value; `words` are git's (`''` where not known). */ -export function gitLogExample(kind: 'gitformat' | 'gitdate', value: string, words: string[], lang: Lang): Example | undefined { +/** + * `git log` output for sample commits, for a `--pretty`/`--format` or `--date` value. `parse` is the parse of + * `log`'s words and `words` their text (undefined where not known), indexed as the parse's words are: the other + * options are read from it, the last one given winning (`--max-count 2`, `-n2`, `-2`). + */ +export function gitLogExample(kind: 'gitformat' | 'gitdate', value: string, parse: ArgvParse, words: (string | undefined)[], lang: Lang): Example | undefined { const s = pick(lang); - const opt = (re: RegExp) => words.map((w) => re.exec(w)?.[1]).filter((v) => v !== undefined).pop(); - const pretty = kind === 'gitformat' ? value : opt(/^--(?:pretty|format)=([\s\S]*)$/) ?? (words.includes('--oneline') ? 'oneline' : undefined); - const date = kind === 'gitdate' ? value : opt(/^--date=([\s\S]*)$/) ?? 'default'; + const last = (...names: string[]): string | undefined => values(parse, ...names).map(({ word, at }) => words[word]?.slice(at ?? 0)).filter((v) => v !== undefined).pop(); + const pretty = kind === 'gitformat' ? value : last('--pretty') ?? (has(parse, '--oneline') ? 'oneline' : undefined); + const date = kind === 'gitdate' ? value : last('--date') ?? 'default'; const now = Date.now(); let commits = sampleCommits(now); - const count = opt(/^(?:-n|--max-count=)(\d+)$/) ?? opt(/^-(\d+)$/) ?? words.find((w, i) => words[i - 1] === '-n' && /^\d+$/.test(w)); - if (count !== undefined) commits = commits.slice(0, Math.max(1, Number(count))); + // `-n N`, `--max-count=N` and `-N` (a pattern option): the last one given. + const counts = [ + ...values(parse, '-n').map(({ word, at }) => ({ word, text: words[word]?.slice(at ?? 0) })), + ...parse.roles.flatMap(({ word, role }) => (role.role === 'pattern-option' ? [{ word, text: role.flag.slice(1) }] : [])), + ].sort((a, b) => a.word - b.word); + const count = counts.pop()?.text; + if (count !== undefined && /^\d+$/.test(count)) commits = commits.slice(0, Math.max(1, Number(count))); try { if (kind === 'gitdate') { return { kind: 'gitLog', diff --git a/src/examples/perlrun.ts b/src/examples/perlrun.ts index 3849cd5c..1b0494a4 100644 --- a/src/examples/perlrun.ts +++ b/src/examples/perlrun.ts @@ -2,6 +2,7 @@ import { sprintf } from './awk'; import { JsRegex, regexExamples, toJsRegex } from './regex'; import { Unsupported as RunnerUnsupported } from './unsupported'; +import { parseOptions, shortKeys, Unsupported as TextUnsupported } from './textfilters'; class Unsupported extends RunnerUnsupported {} @@ -832,36 +833,39 @@ export interface PerlOptions { fieldSep?: string; } -/** Parse perl's command-line switches; the code comes from -e/-E (joined with newlines). */ +/** The switches of perl the examples follow (`-i` and `-w` change nothing they show; `-E` runs code as `-e`). */ +const PERL = shortKeys('eEnplaFiw', { '-E': 'e' }); + +/** + * perl's command-line switches, read with its dictionary spec; the code comes from -e/-E (joined with newlines). + * Undefined for a switch the examples do not follow (`-0`; `-l` with octal digits, `-l0`, which ends each print with that + * character instead of a newline), and a word that is not literal. + */ export function perlOptions(words: (string | undefined)[]): { opts: PerlOptions; code: string } | undefined { + let p; + try { + p = parseOptions('perl', words, PERL); + } catch (e) { + if (e instanceof TextUnsupported) return undefined; + // A bug in the parse: let it show. + /* c8 ignore next */ + throw e; + } + // (A first operand that is not literal may be a switch, or the script.) + if (p.operands.length && p.operands[0] === undefined) return undefined; const opts: PerlOptions = {}; const code: string[] = []; - for (let i = 0; i < words.length; i++) { - const w = words[i]; - if (w === undefined) return undefined; - if (!w.startsWith('-') || w === '-') break; - if (w === '--') break; - for (let k = 1; k < w.length; k++) { - const c = w[k]; - if (c === 'e' || c === 'E') { - const v = k + 1 < w.length ? w.slice(k + 1) : words[++i]; - if (v === undefined) return undefined; - code.push(v); - break; - } - if (c === 'n') opts.loop = opts.loop ?? 'n'; - else if (c === 'p') opts.loop = 'p'; - else if (c === 'l') { opts.chompL = true; if (/^\d/.test(w.slice(k + 1))) return undefined; } - else if (c === 'a') { opts.autosplit = true; opts.loop = opts.loop ?? 'n'; } - else if (c === 'F') { - // -F implies -a and -n (perl 5.20+). - opts.fieldSep = w.slice(k + 1).replace(/^\/(.*)\/$/, '$1').replace(/^(['"])(.*)\1$/, '$2'); - opts.autosplit = true; - opts.loop = opts.loop ?? 'n'; - break; - } else if (c === 'i') { break; } - else if (c === 'w') continue; - else return undefined; + for (const [c, v] of p.opts) { + if (c === 'e') code.push(v); + else if (c === 'n') opts.loop = opts.loop ?? 'n'; + else if (c === 'p') opts.loop = 'p'; + else if (c === 'l') { if (v) return undefined; opts.chompL = true; } + else if (c === 'a') { opts.autosplit = true; opts.loop = opts.loop ?? 'n'; } + else if (c === 'F') { + // -F implies -a and -n (perl 5.20+). + opts.fieldSep = v.replace(/^\/(.*)\/$/, '$1').replace(/^(['"])(.*)\1$/, '$2'); + opts.autosplit = true; + opts.loop = opts.loop ?? 'n'; } } return code.length ? { opts, code: code.join('\n') } : undefined; diff --git a/src/examples/pipeline.ts b/src/examples/pipeline.ts index 8151f943..e278a4cb 100644 --- a/src/examples/pipeline.ts +++ b/src/examples/pipeline.ts @@ -5,11 +5,11 @@ import { perlExample, perlOptions, runPerl } from './perlrun'; import { grepWrap, JsRegex, literalRegex, regexExamples, toJsRegex } from './regex'; import { runSed, sedExample } from './sed'; import { bashPrintf, ShellWordsUnsupported } from './shellwords'; -import { FILTERS, parseOptions, runSeq, sampleInput, Unsupported } from './textfilters'; +import { FILTERS, parseOptions, runSeq, sampleInput, shortKeys, sortsUnique, Unsupported } from './textfilters'; import { runXargs, showCommand } from './runtime'; -import { type ArgvParse, interpret } from '../argv/interpret'; +import type { ArgvParse } from '../argv/interpret'; import { operands } from '../argv/query'; -import { canonicalName, impliedWords, lookupCommand } from '../dict/aliases'; +import { canonicalName, impliedWords } from '../dict/aliases'; export interface Stage { cmd: string; @@ -23,22 +23,12 @@ type Runner = (input: string[]) => string[]; // ---------------------------------------------------------------- grep -const GREP = { - flags: 'ivcnowxEFPGsqHhz', - values: 'em', - long: { - 'ignore-case': 'i', 'invert-match': 'v', count: 'c', 'line-number': 'n', 'only-matching': 'o', 'word-regexp': 'w', 'line-regexp': 'x', - 'extended-regexp': 'E', 'fixed-strings': 'F', 'perl-regexp': 'P', 'basic-regexp': 'G', 'no-messages': 's', quiet: 'q', silent: 'q', - regexp: 'e', 'max-count': 'm', 'no-filename': 'h', 'with-filename': 'H', - }, -}; +/** The options of grep the simulation follows. */ +const GREP = shortKeys('ivcnowxEFPGsqhem'); function grepRegexes(words: (string | undefined)[], relax = false): { res: JsRegex[]; p: ReturnType } { - // (`grep -E -F`: grep refuses the line, as the explanation says.) - if (interpret(lookupCommand('grep')!, words.map((w) => (w === undefined ? {} : { value: w })), { name: 'grep' }).refused) throw new Unsupported('conflicting matchers'); - const p = parseOptions(words, GREP); + const p = parseOptions('grep', words, GREP); const has = (c: string) => p.opts.some(([n]) => n === c); - if (has('z') || has('H')) throw new Unsupported('grep option'); const patterns = p.opts.filter(([n]) => n === 'e').map(([, v]) => v); if (!patterns.length) { const first = p.operands.shift(); @@ -125,34 +115,19 @@ function grepSample(words: (string | undefined)[], more = 0): string[] | undefin // ---------------------------------------------------------------- other runners +/** The options of sed the simulation follows (`-i`, `-s` and `-u` change nothing it shows). */ +const SED = shortKeys('enEisu'); + function sedArgs(words: (string | undefined)[]): { script: string; quiet: boolean; extended: boolean } { - const scripts: string[] = []; - let quiet = false; - let extended = false; - const operands: (string | undefined)[] = []; - for (let i = 0; i < words.length; i++) { - const w = words[i]; - if (w === undefined) throw new Unsupported('non-literal'); - if (w === '-e' || w === '--expression') { const v = words[++i]; if (v === undefined) throw new Unsupported('-e'); scripts.push(v); continue; } - if (w.startsWith('--expression=')) { scripts.push(w.slice(13)); continue; } - if (w === '-n' || w === '--quiet' || w === '--silent') { quiet = true; continue; } - if (w === '-E' || w === '-r' || w === '--regexp-extended') { extended = true; continue; } - if (/^-[nErse]+$/.test(w)) { - if (w.includes('n')) quiet = true; - if (/[Er]/.test(w)) extended = true; - if (w.endsWith('e')) { const v = words[++i]; if (v === undefined) throw new Unsupported('-e'); scripts.push(v); } - continue; - } - if (/^-i/.test(w) || w === '-s' || w === '-u') continue; - if (w.startsWith('-')) throw new Unsupported(`sed ${w}`); - operands.push(w); - } + if (words.includes(undefined)) throw new Unsupported('non-literal'); + const p = parseOptions('sed', words, SED); + const scripts = p.opts.filter(([n]) => n === 'e').map(([, v]) => v); if (!scripts.length) { - const first = operands.shift(); + const first = p.operands[0]; if (first === undefined) throw new Unsupported('script'); scripts.push(first); } - return { script: scripts.join('\n'), quiet, extended }; + return { script: scripts.join('\n'), quiet: p.opts.some(([n]) => n === 'n'), extended: p.opts.some(([n]) => n === 'E') }; } function awkArgs(words: (string | undefined)[]): { program: string; fs?: string; vars: Record; assigns: Record } { @@ -163,7 +138,7 @@ function awkArgs(words: (string | undefined)[]): { program: string; fs?: string; } function runWc(words: (string | undefined)[], input: string[]): string[] { - const p = parseOptions(words, { flags: 'lwcmL', values: '', long: { lines: 'l', words: 'w', bytes: 'c', chars: 'm' } }); + const p = parseOptions('wc', words, shortKeys('lwcmL')); if (p.operands.length) throw new Unsupported('wc files'); const text = input.map((l) => l + '\n').join(''); const counts: Record = { @@ -178,6 +153,9 @@ function runWc(words: (string | undefined)[], input: string[]): string[] { return [order.map((c) => String(counts[c]).padStart(7)).join(' ')]; } +/** Whether the words give no option (else Unsupported) and no file but `-` (or a word not literal). */ +const readsOnlyStdin = (cmd: string, words: (string | undefined)[]): boolean => parseOptions(cmd, words, {}).operands.every((x) => x === '-' || x === undefined); + /** A runner for one stage, or undefined if the command is not supported. */ export function runner(stage: Stage): Runner | undefined { const w = stage.words; @@ -212,11 +190,11 @@ export function runner(stage: Stage): Runner | undefined { return out; }; } - case 'cat': case 'tee': - if (stage.cmd === 'cat' && w.some((x) => x !== '-' && x !== undefined && !x.startsWith('-'))) return undefined; - return (input) => input; - case 'tac': return (input) => [...input].reverse(); - case 'rev': return (input) => input.map((l) => [...l].reverse().join('')); + case 'tee': return (input) => input; + // (No option of these is followed: `cat -n` numbers the lines. A file operand is read instead of the pipe.) + case 'cat': return readsOnlyStdin('cat', w) ? (input) => input : undefined; + case 'tac': return readsOnlyStdin('tac', w) ? (input) => [...input].reverse() : undefined; + case 'rev': return readsOnlyStdin('rev', w) ? (input) => input.map((l) => [...l].reverse().join('')) : undefined; case 'wc': return (input) => runWc(w, input); case 'xargs': return (input) => runXargs(w, input.map((l) => l + '\n').join('')).map(showCommand); } @@ -332,7 +310,8 @@ export function pipelineExample(stages: Stage[], readsFile: boolean, given?: str let rest = stages; const src = !readsFile ? sourceOutput(stages[0]) : undefined; const first = stages[0]; - const firstIsCatFile = first.cmd === 'cat' && first.words.some((x) => x === undefined || (x !== '-' && !x.startsWith('-'))); + // `cat FILE | …`: the stage shows the file. cat with an option (`cat -n f`) throws Unsupported, as its runner declines it. + const firstIsCatFile = first.cmd === 'cat' && parseOptions('cat', first.words, {}).operands.some((x) => x !== '-'); const steps: PipelineStep[] = []; if (src) { steps.push({ stage: first, output: src }); @@ -340,7 +319,8 @@ export function pipelineExample(stages: Stage[], readsFile: boolean, given?: str } else { // The data comes from a file: invent it for the first stage that processes it. const consumer = firstIsCatFile ? stages[1] : first; - const wantsDuplicates = stages.some((s) => s.cmd === 'uniq' || (s.cmd === 'sort' && s.words.some((x) => /^-[a-zA-Z]*u/.test(x ?? '')))); + const wantsDuplicates = stages.some((s) => s.cmd === 'uniq' || (s.cmd === 'sort' && sortsUnique(s.words))); + // (A sort the simulation cannot follow throws Unsupported there: it gives no example anyway.) const build = (more: number): string[] | undefined => { let lines = given ?? sampleFor(consumer, more); // Duplicates make uniq, sort -u and counting visible further down. diff --git a/src/examples/runtime.ts b/src/examples/runtime.ts index c4aaa797..bd88b6f3 100644 --- a/src/examples/runtime.ts +++ b/src/examples/runtime.ts @@ -1,14 +1,21 @@ /** Simulations of xargs, simple test conditions and `read` word splitting, for worked examples. */ import { globExamples, globMatches } from './shellwords'; -import { parseOptions, Unsupported } from './textfilters'; +import { parseOptions, Parsed, shortKeys, Unsupported } from './textfilters'; // ---------------------------------------------------------------- xargs -const XARGS = { - flags: '0rtpx', - values: 'nLIdPsE', - long: { null: '0', 'no-run-if-empty': 'r', verbose: 't', interactive: 'p', exit: 'x', 'max-args': 'n', 'max-lines': 'L', replace: 'I', delimiter: 'd', 'max-procs': 'P', 'max-chars': 's', eof: 'E' }, -}; +/** The options of xargs the examples follow (`-t`, `-x` and `-P` change nothing they show). */ +const XARGS = shortKeys('0rtxnLIdPil'); + +/** + * xargs' options, read with its dictionary spec, and the command it runs (`echo` when none is given). `-i` and `-l`, + * whose value is only attached (`-i@`, `-l2`), are `-I {}` and `-L 1` without one. + */ +function xargsOptions(words: (string | undefined)[]): { p: Parsed; command: (string | undefined)[] } { + const p = parseOptions('xargs', words, XARGS); + p.opts = p.opts.map(([n, v]) => (n === 'i' ? ['I', v || '{}'] : n === 'l' ? ['L', v || '1'] : [n, v])); + return { p, command: p.command === undefined ? ['echo'] : words.slice(p.command) }; +} /** Split xargs input the default way: blanks and newlines separate, quotes and backslashes group. */ function xargsTokens(text: string): string[] { @@ -41,16 +48,10 @@ function xargsTokens(text: string): string[] { /** The command lines xargs runs for `input` (the text on its stdin). */ export function runXargs(words: (string | undefined)[], input: string): string[][] { - // xargs options end at the first non-option word (the command). - const firstCmd = words.findIndex((w, i) => w === undefined || (!w.startsWith('-') && !/^-[nLIdPsE]$/.test(words[i - 1] ?? ''))); - const optWords = firstCmd < 0 ? words : words.slice(0, firstCmd); - const p = parseOptions(optWords, XARGS); - if (p.operands.length) throw new Unsupported('unexpected operand'); - const cmd = firstCmd < 0 ? ['echo'] : words.slice(firstCmd); + const { p, command: cmd } = xargsOptions(words); if (cmd.some((w) => w === undefined)) throw new Unsupported('non-literal command'); const command = cmd as string[]; const get = (c: string) => [...p.opts].reverse().find(([n]) => n === c)?.[1]; - if (get('s') !== undefined || get('E') !== undefined || p.opts.some(([n]) => n === 'p')) throw new Unsupported('xargs option'); const replace = get('I'); const delim = p.opts.some(([n]) => n === '0') ? '\0' : get('d')?.replace(/^\\n$/, '\n').replace(/^\\t$/, '\t'); let items: string[]; @@ -93,7 +94,7 @@ export function showCommand(words: string[]): string { /** Input that shows how xargs groups items, and the commands that result. */ export function xargsExample(words: (string | undefined)[]): { input: string; commands: string[] } | undefined { try { - const p = parseOptions(words.slice(0, Math.max(0, words.findIndex((w, i) => w === undefined || (!w.startsWith('-') && !/^-[nLIdPsE]$/.test(words[i - 1] ?? ''))))) as string[], XARGS); + const { p } = xargsOptions(words); const get = (c: string) => p.opts.find(([n]) => n === c)?.[1]; const files = ['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt']; let input: string; diff --git a/src/examples/textfilters.ts b/src/examples/textfilters.ts index 91314aa5..7e39aaa0 100644 --- a/src/examples/textfilters.ts +++ b/src/examples/textfilters.ts @@ -4,75 +4,84 @@ */ import { Unsupported as RunnerUnsupported } from './unsupported'; +import { type ArgRole, type ArgvParse, interpret } from '../argv/interpret'; +import { lookupCommand } from '../dict/aliases'; + export class Unsupported extends RunnerUnsupported {} -/** Option names → whether they take a value. Long options map to their short form. */ -interface OptSpec { - flags: string; - values: string; - long?: Record; -} +/** + * The options a simulator follows: a name of the option in the command's dictionary spec (any of its names) → the + * key it has in `Parsed.opts`. The key `_` marks an option whose value the simulator does not read, so it may be not literal. + */ +export type Supported = Record; export interface Parsed { - /** Options in order: [name, value] (value '' for flags). */ + /** Options in order: [key, value] (value '' for flags and for an optional value not given). */ opts: [string, string][]; /** Operands (files, numbers, names); undefined when not literal. */ operands: (string | undefined)[]; + /** The index of the word that starts the command this one runs (`xargs … CMD`), if any. */ + command?: number; } -/** GNU-style option parsing: combined flags (-nr), attached values (-k2 -t:), --long[=value], and `--`. */ -export function parseOptions(words: (string | undefined)[], spec: OptSpec): Parsed { - const opts: [string, string][] = []; - const operands: (string | undefined)[] = []; - let ended = false; - for (let i = 0; i < words.length; i++) { - const w = words[i]; - if (w === undefined || ended || w === '-' || !w.startsWith('-')) { - operands.push(w); - continue; - } - if (w === '--') { ended = true; continue; } - if (w.startsWith('--')) { - const eq = w.indexOf('='); - const name = eq < 0 ? w.slice(2) : w.slice(2, eq); - const short = spec.long?.[name]; - if (!short) throw new Unsupported(`--${name}`); - if (spec.values.includes(short)) { - const v = eq < 0 ? words[++i] : w.slice(eq + 1); - if (v === undefined) throw new Unsupported('missing value'); - opts.push([short, v]); - } else opts.push([short, '']); - continue; - } - for (let k = 1; k < w.length; k++) { - const c = w[k]; - if (spec.values.includes(c)) { - const v = k + 1 < w.length ? w.slice(k + 1) : words[++i]; - if (v === undefined) throw new Unsupported('missing value'); - opts.push([c, v]); +/** The parse of a command's words with its dictionary spec, as the explanation reads them; a word that is not literal is undefined. */ +export function parseWords(cmd: string, words: (string | undefined)[]): ArgvParse { + return interpret(lookupCommand(cmd)!, words.map((w) => (w === undefined ? {} : { value: w })), { name: cmd, lookup: lookupCommand }); +} + +/** + * The options and operands of `cmd`'s words, read with its dictionary spec (`parseWords`): clusters (`-nr`), values + * attached or in the next word (`-k2`, `-t :`), `--name=value`, a unique prefix of a long name where the tool takes + * one (`--rev`), and `--`. An option `supported` does not name, one the spec does not know or that is ambiguous, and + * a value that is missing or not literal throw `Unsupported`. `pattern` gives the key and value of an option the + * spec recognises by its form (`head -5`: `-n 5`). + */ +export function parseOptions(cmd: string, words: (string | undefined)[], supported: Supported, pattern?: (flag: string) => [string, string]): Parsed { + const out: Parsed = { opts: [], operands: [] }; + const option = (r: ArgRole, word: number): void => { + if (r.role === 'pattern-option' && pattern) { out.opts.push(pattern(r.flag)); return; } + if (r.role !== 'option') throw new Unsupported(`${r.role} ${words[word]}`); + const key = r.spec.names.map((n) => supported[n]).find((k) => k !== undefined); + if (key === undefined) throw new Unsupported(r.flag); + if (r.missing) throw new Unsupported('missing value'); + out.opts.push([key, r.value ? words[word]!.slice(r.value.attached) : '']); + }; + const parse = parseWords(cmd, words); + // (A line the tool refuses, `grep -E -F`, runs nothing.) + if (parse.refused) throw new Unsupported('refused'); + for (const { word, role } of parse.roles) { + switch (role.role) { + case 'operand': out.operands.push(words[word]); break; + case 'end-of-options': break; + case 'command': out.command = word; break; + case 'cluster': role.letters.forEach((l) => option(l, word)); break; + case 'value': { + const v = words[word]; + // (A value the simulator does not read, key `_`, may be anything: getopt's `-n "$0"`.) + if (v === undefined && out.opts[out.opts.length - 1][0] !== '_') throw new Unsupported('missing value'); + out.opts[out.opts.length - 1][1] = v ?? ''; break; } - if (!spec.flags.includes(c)) throw new Unsupported(`-${c}`); - opts.push([c, '']); + default: option(role, word); } } - return { opts, operands }; + return out; } +/** The short options `-x` of `letters` as keys `x`, and `more`. */ +export const shortKeys = (letters: string, more: Supported = {}): Supported => ({ ...Object.fromEntries([...letters].map((c) => [`-${c}`, c])), ...more }); + const has = (p: Parsed, c: string) => p.opts.some(([n]) => n === c); const value = (p: Parsed, c: string) => [...p.opts].reverse().find(([n]) => n === c)?.[1]; const values = (p: Parsed, c: string) => p.opts.filter(([n]) => n === c).map(([, v]) => v); // ---------------------------------------------------------------- sort -const SORT: OptSpec = { - flags: 'bdfghinrsuVM', - values: 'ktS', - long: { - 'ignore-leading-blanks': 'b', 'dictionary-order': 'd', 'ignore-case': 'f', 'general-numeric-sort': 'g', 'human-numeric-sort': 'h', - 'numeric-sort': 'n', reverse: 'r', stable: 's', unique: 'u', 'version-sort': 'V', key: 'k', 'field-separator': 't', 'buffer-size': 'S', - }, -}; +/** The options of sort the simulation follows (`-S`, the buffer size, changes nothing it shows). */ +const SORT = shortKeys('bdfghinrsuVktS'); + +/** Whether sort's words give `-u` (`-nu`, `--unique`, `--uniq`); throws Unsupported where the simulation cannot follow them. */ +export const sortsUnique = (words: (string | undefined)[]): boolean => has(parseOptions('sort', words, SORT), 'u'); interface SortKey { sf: number; sc: number; ef: number; ec: number; @@ -170,8 +179,7 @@ function compareKey(a: string, b: string, opts: string): number { } export function runSort(words: (string | undefined)[], input: string[]): string[] { - const p = parseOptions(words, SORT); - if (has(p, 'M')) throw new Unsupported('month sort'); + const p = parseOptions('sort', words, SORT); const global = p.opts.filter(([n]) => 'bdfghinrV'.includes(n)).map(([n]) => n).join(''); const sep = value(p, 't'); if (sep !== undefined && [...sep].length !== 1) throw new Unsupported('-t'); @@ -193,15 +201,17 @@ export function runSort(words: (string | undefined)[], input: string[]): string[ // ---------------------------------------------------------------- uniq -const UNIQ: OptSpec = { - flags: 'cdDiuz', - values: 'fsw', - long: { count: 'c', repeated: 'd', 'all-repeated': 'D', 'ignore-case': 'i', unique: 'u', 'skip-fields': 'f', 'skip-chars': 's', 'check-chars': 'w' }, +/** The options of uniq the simulation follows; `--all-repeated` only as `-D` (its method `none`). */ +const UNIQ = shortKeys('cdDiufsw', { '--all-repeated': 'D' }); + +const uniqOptions = (words: (string | undefined)[]): Parsed => { + const p = parseOptions('uniq', words, UNIQ); + if (p.opts.some(([k, v]) => k === 'D' && v !== '' && v !== 'none')) throw new Unsupported('--all-repeated'); + return p; }; export function runUniq(words: (string | undefined)[], input: string[]): string[] { - const p = parseOptions(words, UNIQ); - if (has(p, 'z')) throw new Unsupported('-z'); + const p = uniqOptions(words); const skipF = Number(value(p, 'f') ?? 0); const skipC = Number(value(p, 's') ?? 0); const width = value(p, 'w') !== undefined ? Number(value(p, 'w')) : Infinity; @@ -229,11 +239,8 @@ export function runUniq(words: (string | undefined)[], input: string[]): string[ // ---------------------------------------------------------------- cut -const CUT: OptSpec = { - flags: 'sn', - values: 'bcfdO', - long: { bytes: 'b', characters: 'c', fields: 'f', delimiter: 'd', 'only-delimited': 's', 'output-delimiter': 'O', complement: 'C' }, -}; +/** The options of cut the simulation follows (`-n` changes nothing). */ +const CUT = shortKeys('snbcfd', { '--output-delimiter': 'O', '--complement': 'C' }); function parseList(list: string): (i: number) => boolean { const ranges = list.split(',').map((r) => { @@ -247,8 +254,8 @@ function parseList(list: string): (i: number) => boolean { } export function runCut(words: (string | undefined)[], input: string[]): string[] { - const complement = words.includes('--complement'); - const p = parseOptions(words.filter((w) => w !== '--complement'), { ...CUT, flags: CUT.flags + 'C' }); + const p = parseOptions('cut', words, CUT); + const complement = has(p, 'C'); const pick = (sel: (i: number) => boolean) => (i: number) => (complement ? !sel(i) : sel(i)); const fields = value(p, 'f'); const chars = value(p, 'c') ?? value(p, 'b'); @@ -334,14 +341,15 @@ export function expandSet(set: string, fillTo?: number): string[] { return out; } -const TR: OptSpec = { flags: 'cCdst', values: '', long: { complement: 'c', delete: 'd', 'squeeze-repeats': 's', 'truncate-set1': 't' } }; +/** The options of tr the simulation follows. */ +const TR = shortKeys('cdst'); /** tr over the whole text (lines joined with \n), so sets containing \n behave as in tr. */ export function runTr(words: (string | undefined)[], input: string[]): string[] { - const p = parseOptions(words, TR); + const p = parseOptions('tr', words, TR); const sets = p.operands; if (sets.some((s) => s === undefined) || !sets.length || sets.length > 2) throw new Unsupported('sets'); - const comp = has(p, 'c') || has(p, 'C'); + const comp = has(p, 'c'); const del = has(p, 'd'); const squeeze = has(p, 's'); const text = input.map((l) => l + '\n').join(''); @@ -390,17 +398,17 @@ export function runTr(words: (string | undefined)[], input: string[]): string[] // ---------------------------------------------------------------- head / tail -/** `head -5` / `tail +2` (obsolete forms) become `-n 5` / `-n +2`. */ -function normaliseCount(words: (string | undefined)[]): (string | undefined)[] { - return words.flatMap((w, i) => { - // `-n -2`: the number is the value of -n, not the obsolete form. - if (i > 0 && /^(-n|-c|--lines|--bytes)$/.test(words[i - 1] ?? '')) return [w]; - return w && /^-\d+$/.test(w) ? ['-n', w.slice(1)] : w && /^\+\d+$/.test(w) ? ['-n', w] : [w]; - }); -} +/** The options of head and tail the simulation follows. */ +const HEAD = shortKeys('qvznc'); +const TAIL = shortKeys('fFqvzncs', { '--follow': 'f' }); + +/** `head -5`, `tail -5`, `tail +2` (the old forms the specs recognise) are `-n 5`, `-n +2`. */ +const lineCount = (flag: string): [string, string] => ['n', flag.startsWith('+') ? flag : flag.slice(1)]; + +const countOptions = (cmd: 'head' | 'tail', words: (string | undefined)[]): Parsed => parseOptions(cmd, words, cmd === 'head' ? HEAD : TAIL, lineCount); export function runHead(words: (string | undefined)[], input: string[]): string[] { - const p = parseOptions(normaliseCount(words), { flags: 'qvz', values: 'nc', long: { lines: 'n', bytes: 'c', quiet: 'q', silent: 'q', verbose: 'v' } }); + const p = countOptions('head', words); const c = value(p, 'c'); if (c !== undefined) { const text = input.map((l) => l + '\n').join(''); @@ -415,7 +423,7 @@ export function runHead(words: (string | undefined)[], input: string[]): string[ } export function runTail(words: (string | undefined)[], input: string[]): string[] { - const p = parseOptions(normaliseCount(words), { flags: 'fFqvz', values: 'ncs', long: { lines: 'n', bytes: 'c', follow: 'f', quiet: 'q', silent: 'q', verbose: 'v' } }); + const p = countOptions('tail', words); const c = value(p, 'c'); if (c !== undefined) { const text = input.map((l) => l + '\n').join(''); @@ -432,10 +440,9 @@ export function runTail(words: (string | undefined)[], input: string[]): string[ // ---------------------------------------------------------------- seq, basename, dirname export function runSeq(words: (string | undefined)[]): string[] { - // Negative numbers are operands, not options. - const fixed = words.map((w) => (w && /^-\d/.test(w) ? `\u0000${w}` : w)); - const p = parseOptions(fixed, { flags: 'w', values: 'sf', long: { 'equal-width': 'w', separator: 's', format: 'f' } }); - const nums = p.operands.map((o) => o?.replace('\u0000', '')); + // (A negative number is an operand: the spec says so, as GNU seq reads it.) + const p = parseOptions('seq', words, shortKeys('wsf')); + const nums = p.operands; // seq needs one to three numbers. if (nums.length < 1 || nums.length > 3 || nums.some((n) => n === undefined || !/^-?(\d+\.?\d*|\.\d+)$/.test(n))) throw new Unsupported('seq operands'); const [first, inc, last] = nums.length === 1 ? ['1', '1', nums[0]!] : nums.length === 2 ? [nums[0]!, '1', nums[1]!] : [nums[0]!, nums[1]!, nums[2]!]; @@ -465,7 +472,7 @@ export function runSeq(words: (string | undefined)[]): string[] { } export function runBasename(words: (string | undefined)[]): string[] { - const p = parseOptions(words, { flags: 'az', values: 's', long: { multiple: 'a', suffix: 's', zero: 'z' } }); + const p = parseOptions('basename', words, shortKeys('asz')); if (p.operands.some((o) => o === undefined) || !p.operands.length) throw new Unsupported('operands'); const names = p.operands as string[]; const multiple = has(p, 'a') || has(p, 's'); @@ -479,7 +486,7 @@ export function runBasename(words: (string | undefined)[]): string[] { } export function runDirname(words: (string | undefined)[]): string[] { - const p = parseOptions(words, { flags: 'z', values: '', long: { zero: 'z' } }); + const p = parseOptions('dirname', words, shortKeys('z')); if (p.operands.some((o) => o === undefined) || !p.operands.length) throw new Unsupported('operands'); return (p.operands as string[]).map((n) => { if (/^\/+$/.test(n)) return '/'; @@ -515,7 +522,7 @@ export function sampleInput(cmd: string, words: (string | undefined)[], more = 0 try { switch (cmd) { case 'sort': { - const p = parseOptions(words, SORT); + const p = parseOptions('sort', words, SORT); const global = p.opts.filter(([n]) => 'bdfghinrV'.includes(n)).map(([n]) => n).join(''); const sep = value(p, 't') ?? ' '; const keys = values(p, 'k').map((k) => parseKey(k, global)); @@ -548,13 +555,13 @@ export function sampleInput(cmd: string, words: (string | undefined)[], more = 0 return rows.map((r) => r.join(sep)); } case 'uniq': { - const p = parseOptions(words, UNIQ); + const p = uniqOptions(words); if (has(p, 'i')) return ['apple', 'Apple', 'banana', 'apple', 'cherry', 'cherry']; if (value(p, 'f')) return ['1 apple', '2 apple', '3 banana', '4 apple', '5 cherry', '6 cherry']; return ['apple', 'apple', 'banana', 'apple', 'cherry', 'cherry']; } case 'cut': { - const p = parseOptions(words.filter((w) => w !== '--complement'), { ...CUT, flags: CUT.flags + 'C' }); + const p = parseOptions('cut', words, CUT); const list = value(p, 'f') ?? value(p, 'c') ?? value(p, 'b') ?? '1'; const max = Math.max(...list.split(/[,-]/).filter(Boolean).map(Number)); if (value(p, 'f') !== undefined) { @@ -573,10 +580,10 @@ export function sampleInput(cmd: string, words: (string | undefined)[], more = 0 return ['abcdefghijklmnopqrstuvwxyz'.slice(0, len), '0123456789abcdefghijklmn'.slice(0, len)]; } case 'tr': { - const p = parseOptions(words, TR); + const p = parseOptions('tr', words, TR); const set1 = p.operands[0] !== undefined ? expandSet(p.operands[0]) : []; // A complement keeps the set and changes everything else, so include other characters. - if (has(p, 'c') || has(p, 'C')) return ['Hello, World! #123']; + if (has(p, 'c')) return ['Hello, World! #123']; if (set1.includes('\n')) return ['one', 'two', 'three']; if (set1.includes('\r')) return ['line 1\r', 'line 2\r']; if (has(p, 's') && set1.includes(' ')) return ['too many spaces']; @@ -588,7 +595,7 @@ export function sampleInput(cmd: string, words: (string | undefined)[], more = 0 } case 'head': case 'tail': { - const p = parseOptions(normaliseCount(words), { flags: 'fFqvz', values: 'ncs', long: { lines: 'n', bytes: 'c', follow: 'f' } }); + const p = countOptions(cmd, words); if (value(p, 'c') !== undefined) return ['abcdefghij', 'klmnopqrst']; const n = Math.abs(Number(value(p, 'n') ?? '10')); const count = Math.min(Math.max(n + 3, 6), 14) + more; diff --git a/test/branches.test.ts b/test/branches.test.ts index 5e7bbcd2..773e412c 100644 --- a/test/branches.test.ts +++ b/test/branches.test.ts @@ -257,7 +257,6 @@ describe('branches no other test takes', () => { it('xargs, tests and read: the cases the simulation leaves out', () => { assert.throws(() => runXargs([], "a 'b"), Unsupported); - assert.throws(() => runXargs(['-'], ''), Unsupported); assert.throws(() => runXargs(['echo', undefined], 'a'), Unsupported); assert.throws(() => runXargs(['-s', '10'], 'a'), Unsupported); assert.deepStrictEqual(runXargs(['-r', 'echo'], ''), []); @@ -716,6 +715,9 @@ describe('the analyzer: branches no other test takes', () => { return [en, ja].map((e) => [...e.context.map((c) => c.label), ...e.notes, ...allTokens(e.items).map((t) => `${t.label}: ${t.explanation}`), ...e.examples.map((x) => x.title)].join('\n')); }; const cases: [string, number?, Shell?, ExplainOptions?][] = [ + // `~` read by the shell state with HOME not known (`tilde` in src/semantics/state.ts, its `?? undefined`): only + // the seeded fuzz took that branch, and new snippets changed its inputs. + ['~'], ['HOME=/h; [[ ~root == /h ]]'], ['[[ ~+ == x ]]'], ['echo ${a[10]} ${a[11]} ${a[12]} ${a[13]} ${a[1]} ${a[2]} ${a[3]} ${a[-11]} ${a[-2]} ${s:10:1} ${s:11} ${s:21:1} ${s:22:1} ${s:23}'], ['echo "$x" | grep -c a', 0, undefined, { idioms: false, pitfalls: false }], ['until false; do\n echo a\ndone', 1], ['((\n x = 1\n))', 1], ['a=1 b=2'], ['(( x ? 1 : 2 )); (( x-- )); (( (1 + 2) ))'], ["echo $'a\\tb'"], @@ -736,7 +738,7 @@ describe('the analyzer: branches no other test takes', () => { ['{ cat; } <<< x 2>&1 > f'], ['case $x in\n a |\n b) :;;\nesac', 1], ['case $x in\n (\n a) :;;\nesac', 1], ['case $x in\n a)\n echo;;\nesac', 2], ['x=1 \\\n && case $y in a) :;; esac'], ['cat < f | sort'], ['grep a | sort'], ['perl -ne "$x"'], ['seq 3 | while read -r n; do t=$n; done; echo "$t"'], ['seq 3 | read v; echo "$v"', 0, 'dash'], ['seq 3 | { read v; }; echo "$v"'], ['seq 3 | for i in 1; do s=1; done; echo $s'], ['echo a | X=1 cat; echo $X'], ['f() { echo | while read -r l; do local q=1; done; echo $q; }'], ['echo | while read -r -d x l; do :; done; echo $l'], diff --git a/test/examples-awk.test.ts b/test/examples-awk.test.ts index c8527376..1b00e9cf 100644 --- a/test/examples-awk.test.ts +++ b/test/examples-awk.test.ts @@ -81,7 +81,7 @@ describe('awk command lines (options and name=value operands) agree with gawk', assert.strictEqual(awkInvocation(['-v', 'x=a\\', 'BEGIN{}']).vars.x, 'a\\'); }); it('refuses what it cannot follow', () => { - for (const words of [['{print}', 'f', 'x=1'], ['-v', 'x=\\q', '{print}'], ['{print}', 'x=\\q'], ['-F', '\\d', '{print}'], ['--posix', '{print}'], ['-v', 'x', '{print}'], ['-F'], [undefined, 'x']]) { + for (const words of [['{print}', 'f', 'x=1'], ['-v', 'x=\\q', '{print}'], ['{print}', 'x=\\q'], ['-F', '\\d', '{print}'], ['--posix', '{print}'], ['-v', 'x', '{print}'], ['-F'], [undefined, 'x'], ['-xF:', '{print}']]) { assert.ok(awkInvocation(words).unsupported !== undefined, JSON.stringify(words)); } const f = awkInvocation(['-f', 'p.awk', 'x=1']); diff --git a/test/examples-gitlog.test.ts b/test/examples-gitlog.test.ts index 39211f0f..f5974c29 100644 --- a/test/examples-gitlog.test.ts +++ b/test/examples-gitlog.test.ts @@ -43,3 +43,28 @@ describe('git log formats agree with git', () => { }); } }); + +describe('the git log example shows as many commits as git', () => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { Analyzer } = require('../src/analyzer'); + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { createUnbashParser } = require('../src/syntax/adapters/unbash'); + let dir = ''; + const env = { GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1', GIT_AUTHOR_NAME: 'a', GIT_AUTHOR_EMAIL: 'a@x', GIT_COMMITTER_NAME: 'a', GIT_COMMITTER_EMAIL: 'a@x' }; + const git = (args: string[]) => execFileSync('git', args, { cwd: dir, env: { ...process.env, ...env } }).toString(); + before(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gl-n-')); + git(['init', '-q', '-b', 'main']); + for (let i = 0; i < sampleCommits(Date.now()).length; i++) git(['commit', '-q', '--allow-empty', '-m', `c${i}`]); + }); + after(() => fs.rmSync(dir, { recursive: true, force: true })); + // The count is read as git reads it: `--max-count N` (a separate value), `-nN`, `-N`, the last one winning. + for (const args of [['--max-count', '2'], ['--max-count=2'], ['-n2'], ['-n', '1'], ['-2'], ['-n', '3', '-1'], ['-1', '-n', '2'], ['--date', 'short', '-n', '2'], []]) { + const line = ['git log', ...args, '--format=%h'].join(' '); + it(line, () => { + const ex = new Analyzer('en').explainLine(createUnbashParser().parse(line), 0).examples.find((e: { kind: string }) => e.kind === 'gitLog'); + assert.ok(ex, 'has an example'); + assert.strictEqual(ex.sections[0].lines.length, git(['log', ...args, '--format=%h']).trim().split('\n').length); + }); + } +}); diff --git a/test/examples-pipeline.test.ts b/test/examples-pipeline.test.ts index 684d7dd6..1226de1d 100644 --- a/test/examples-pipeline.test.ts +++ b/test/examples-pipeline.test.ts @@ -39,6 +39,9 @@ describe('pipeline examples agree with bash', () => { [['awk', '-v', 'x=1', '{ print x, $0 }'], ['cat']], [['awk', '-vx=2', '--', '{ print x }'], ['cat']], [['seq', '3'], ['wc']], [['seq', '3'], ['wc', '-l', '-w']], [['seq', '3'], ['wc', '-m']], [['seq', '3'], ['cat', '-']], [['seq', '3'], ['tee']], + // Options read as the explanation reads them: long prefixes, a value that looks like an option, clusters. + [['grep', '--inv', 'a'], ['cat']], [['grep', '-e', '-v', '-e', 'a'], ['cat']], [['sed', '--expr', 's/a/X/'], ['cat']], [['sed', '-ns', '/e/p'], ['cat']], + [['sort', '--rev'], ['uniq']], [['seq', '3'], ['wc', '--li']], [['perl', '-lne', 'print if /e/'], ['cat']], ]; for (const words of cases) { const stages: Stage[] = words.map(([cmd, ...w]) => ({ cmd, words: w, text: [cmd, ...w].join(' ') })); @@ -104,6 +107,37 @@ describe('pipeline examples agree with bash', () => { ]; for (const stages of none.filter((x) => x.length > 1)) assert.strictEqual(pipelineExample(stages.map(st), false), undefined, JSON.stringify(stages)); }); + it('declines what the tools refuse', () => { + // GNU grep: "conflicting matchers specified" (status 2). + const refused = [['grep', '-E', '-F', 'a'], ['grep', '-G', '--perl', 'a']]; + for (const words of refused) { + const stages = [['seq', '3'], words].map(([cmd, ...w]): Stage => ({ cmd, words: w, text: [cmd, ...w].join(' ') })); + assert.strictEqual(pipelineExample(stages, false), undefined, words.join(' ')); + assert.notStrictEqual(run('/usr/bin/grep', words.slice(1), { input: '1\n' }).status, 0, words.join(' ')); + } + }); + // cat, tac and rev pass on their input (reversed) only without options and files: `cat -n` numbers the lines. + it('declines cat, tac and rev with an option or a file, whose output is not the input as passed on', () => { + for (const words of [['cat', '-n'], ['cat', '--number'], ['tac', '-s', '2'], ['tac', 'f'], ['rev', 'f']]) { + const stages = [['seq', '3'], words].map(([cmd, ...w]): Stage => ({ cmd, words: w, text: [cmd, ...w].join(' ') })); + assert.strictEqual(pipelineExample(stages, false), undefined, words.join(' ')); + const real = run(words[0], words.slice(1), { input: '1\n2\n3\n' }); + assert.ok(real.status !== 0 || String(real.stdout) !== '1\n2\n3\n', words.join(' ')); + } + for (const words of [['cat'], ['cat', '-'], ['tac'], ['rev']]) { + const stages = [['seq', '3'], words].map(([cmd, ...w]): Stage => ({ cmd, words: w, text: [cmd, ...w].join(' ') })); + assert.ok(pipelineExample(stages, false), words.join(' ')); + } + // A first `cat FILE` shows the file as it is; with an option (`cat -n f`, numbered lines) there is no example. + const st = (...w: string[]): Stage => ({ cmd: w[0], words: w.slice(1), text: w.join(' ') }); + for (const first of [st('cat', '-n', 'f'), st('cat', '--number', 'f'), st('cat', '-A', 'f')]) assert.strictEqual(pipelineExample([first, st('sort')], false), undefined, first.text); + assert.ok(pipelineExample([st('cat', 'f'), st('sort')], false)); + assert.ok(pipelineExample([st('cat', '--', '-n'), st('sort')], false), 'cat -- -n reads the file -n'); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cat-n-')); + fs.writeFileSync(path.join(tmp, 'f'), 'b\na\n'); + assert.strictEqual(String(run('cat', ['--number', path.join(tmp, 'f')]).stdout), ' 1\tb\n 2\ta\n'); + fs.rmSync(tmp, { recursive: true, force: true }); + }); it('starts from sample data when the first stage is not a source', () => { const st = (cmd: string, ...words: (string | undefined)[]): Stage => ({ cmd, words, text: [cmd, ...words].join(' ') }); for (const first of [st('echo', '-n', 'x'), st('printf', '-v', 'x'), st('echo', undefined), st('printf')]) { diff --git a/test/examples-runtime.test.ts b/test/examples-runtime.test.ts index aee09459..d1a2d5c4 100644 --- a/test/examples-runtime.test.ts +++ b/test/examples-runtime.test.ts @@ -9,6 +9,8 @@ describe('xargs agrees with GNU xargs', () => { [['-n1', '-P4'], ['gzip']], [['-r'], ['ls']], [['-n', '3'], ['cp', '-t', 'dest']], [['--max-args=2'], ['echo']], // With `-I` and a delimiter, each item is a run of its own. [['-0', '-n', '1', '-I{}'], ['rm', '-f', '{}']], [['-0', '-I{}'], ['mv', '{}', '{}.bak']], [['-d', ',', '-I{}'], ['echo', '{}']], + // `-i`/`--replace` and `-l`/`--max-lines` take a value only attached: the next word is the command (`-I {}`, `-L 1`). + [['--replace'], ['echo', '{}']], [['-i'], ['mv', '{}', '{}.bak']], [['-i@'], ['echo', '@']], [['--max-lines'], ['echo']], [['-l'], ['echo']], [['--max-l=2'], ['echo']], ]; for (const [opts, cmd] of cases) { it([...opts, ...cmd].join(' ') || '(no options)', () => { diff --git a/test/examples-textfilters.test.ts b/test/examples-textfilters.test.ts index d1b5188f..45932cac 100644 --- a/test/examples-textfilters.test.ts +++ b/test/examples-textfilters.test.ts @@ -25,6 +25,9 @@ const cases: string[][] = [ ['seq', '5'], ['seq', '2', '5'], ['seq', '1', '3', '10'], ['seq', '-w', '8', '11'], ['seq', '-s', ',', '3'], ['seq', '0', '0.5', '2'], ['seq', '10', '-3', '1'], ['seq', '-w', '-2', '2'], ['basename', '/a/b/report.txt', '.txt'], ['basename', '-a', '/x/y', 'z/w/'], ['basename', '-s', '.sh', 'a.sh', 'b/c.sh'], ['basename', '/'], ['basename', '.sh', '.sh'], ['dirname', '/a/b/c.txt'], ['dirname', 'file'], ['dirname', '/'], ['cut', '-b', '2-4'], ['cut', '-d,', '-f9'], ['uniq', '-w', '3'], ['dirname', '/a/b/', 'x/y', '/z'], ['dirname', '//a//b//'], + // Options read as the explanation reads them (the dictionary spec): long prefixes, other names of an option. + ['sort', '--rev'], ['sort', '--uniq'], ['cut', '-d,', '--comp', '-f2'], ['cut', '-n', '-f2'], ['tr', '-C', '[:alnum:]\\n', '_'], ['uniq', '--all-repeated'], + ['uniq', '--all-repeated=none'], ['head', '-q', '-n', '3'], ['tail', '--lin=2'], ['seq', '-.5', '1'], ['basename', '--suf=.sh', 'a.sh'], ]; describe('coreutils examples agree with GNU coreutils', () => { @@ -59,6 +62,12 @@ describe('coreutils example input', () => { for (const args of [[], ['-w'], ['-s', ','], ['1', '2', '3', '4']]) assert.strictEqual(commandExample('seq', args), undefined, args.join(' ')); assert.strictEqual(commandExample('basename', []), undefined); assert.throws(() => runSort(['--bogus'], [])); + // GNU seq reads options only before the first number (`seq 1 -w 3`: invalid floating point argument: '-w'), and + // uniq's other grouping methods are not `-D`. + for (const [cmd, ...args] of [['seq', '1', '-w', '3'], ['uniq', '--all-repeated=prepend']]) { + assert.strictEqual(commandExample(cmd, args), undefined, args.join(' ')); + } + assert.notStrictEqual(run('seq', ['1', '-w', '3']).status, 0); }); }); @@ -98,7 +107,7 @@ describe('coreutils filters agree with GNU coreutils on fixed input', () => { [() => runSeq(['1', '0', '3']), 'zero'], [() => runSeq(['1', '100000']), 'too long'], [() => runSeq(['-f', '%g', '3']), '-f'], ]; for (const [f, msg] of fails) assert.throws(f, (e: Error) => e instanceof Unsupported && e.message.includes(msg), msg); - assert.deepStrictEqual(parseOptions(['--', '-x'], { flags: 'x', values: '' }).operands, ['-x']); + assert.deepStrictEqual(parseOptions('sort', ['--', '-x'], {}).operands, ['-x']); assert.strictEqual(sampleInput('wc', []), undefined); assert.strictEqual(commandExample('nope', []), undefined); assert.ok(sampleInput('cut', ['-s', '-f', '2'])!.includes('no delimiter here')); diff --git a/test/fixtures/snippets.json b/test/fixtures/snippets.json index e41ac60d..a729bb76 100644 --- a/test/fixtures/snippets.json +++ b/test/fixtures/snippets.json @@ -173,6 +173,7 @@ "FOO=1 time ls", "GLOBIGNORE='*.log'; ls *", "GLOBIGNORE='*.log'; ls *.log", + "HOME=/h; [[ ~root == /h ]]", "IFS= read -r a b", "IFS=\"$x\" read a", "IFS=: read -r a b", @@ -290,6 +291,7 @@ "[[ a == ]]", "[[ a b ]]", "[[ a$b == ab ]]", + "[[ ~+ == x ]]", "[[.a.]]", "[[:foo:]]", "[abc", @@ -1074,6 +1076,7 @@ "yq 'env(HOME)' f.yml", "yq --output-format=json '.a' f.yml", "yq -i '.a = 1' f.yml", + "yq -o \"$f\" '.a' f.yml", "yq -o json .a f.yml", "yq -o=xml '.a' f.yml", "yq -oj '.a' f.yml", @@ -1104,5 +1107,6 @@ "{a b}", "{a: .b | .}", "{echo,hi}", - "{ls,-l}" + "{ls,-l}", + "~" ] diff --git a/test/options-fixes.test.ts b/test/options-fixes.test.ts index 7e7e5394..2236630f 100644 --- a/test/options-fixes.test.ts +++ b/test/options-fixes.test.ts @@ -78,6 +78,23 @@ const roles: [string, Record][] = [ ['ip n show', { n: /neighbour/ }], ['ip addr flush dev eth0', { flush: /Remove every address/ }], ['pkill -f nginx', { nginx: /pattern/ }], + // GNU seq: a negative number is the first operand, and options end there (checked below). + ['seq -w -2 2', { '-2': /^first/, 2: /^last/ }], + ['seq 10 -3 1', { '-3': /^increment/ }], + // GNU tail: `+2` is `-n +2`. + ['tail +2 f', { '+2': /from line 2/ }], + // gawk and perl read options only before the program (or perl's first file). + ["awk 'NR>1' f -v", { '-v': /target file/ }], + ["perl -ne 'print' f -l", { '-l': /target file/ }], + // GNU tr's `-C` is `-c`; GNU cut accepts `-n` and ignores it. + ['tr -C a x', { '-C': /complement/ }], + ['cut -n -f1', { '-n': /ignored/ }], + // perl's `-l` and `-0` take octal digits attached; other letters after them are switches (checked below). + ["perl -l0 -e 'print 1'", { 0: /Value `0` for `-l`/ }], + ["perl -0ne 'print'", { '-0ne': /`-0` `-n` `-e`/ }], + ["perl -lne 'print'", { '-lne': /`-l` `-n` `-e`/ }], + // util-linux getopt reads its own options only before the first operand: `-u` is an argument it parses (checked below). + ['getopt -o a x -u', { '-u': /arguments to parse/ }], ]; describe('option dictionary fixes (audit)', () => { @@ -129,6 +146,19 @@ describe('option dictionary fixes (audit)', () => { fs.writeFileSync(path.join(tmp, 'in.txt'), 'a\na\nb\n'); run('uniq', ['in.txt', 'out.txt']); assert.strictEqual(fs.readFileSync(path.join(tmp, 'out.txt'), 'utf8'), 'a\nb\n'); + assert.strictEqual(run('seq', ['-w', '-2', '2']), '-2\n-1\n00\n01\n02'); + assert.strictEqual(run('seq', ['10', '-3', '1']), '10\n7\n4\n1'); + fs.writeFileSync(path.join(tmp, 'f'), 'a\nb\nc\n'); + assert.strictEqual(run('tail', ['+2', 'f']), 'b\nc'); + assert.strictEqual(run('gawk', ['BEGIN { print ARGV[1], ARGV[2] }', 'f', '-v']), 'f -v'); + assert.strictEqual(run('perl', ['-e', 'print "@ARGV"', 'f', '-l']), 'f -l'); + assert.strictEqual(execFileSync('tr', ['-C', 'a', 'x'], { input: 'abc', encoding: 'utf8' }), 'axx'); + assert.strictEqual(run('cut', ['-n', '-f1', 'f']), 'a\nb\nc'); + assert.strictEqual(run('perl', ['-l0', '-e', 'print 1']), '1\u0000'); + assert.strictEqual(run('perl', ['-0ne', 'print', 'f']), 'a\nb\nc'); + assert.strictEqual(run('perl', ['-lne', 'print length', 'f']), '1\n1\n1'); + // (getopt parses `x -u` with `-o a`: "invalid option -- 'u'", status 1; were `-u` its own, it would quote nothing and succeed.) + assert.throws(() => execFileSync('getopt', ['-o', 'a', 'x', '-u'], { stdio: 'pipe' })); }); }); diff --git a/test/yq.test.ts b/test/yq.test.ts index d4605f62..2df7b61c 100644 --- a/test/yq.test.ts +++ b/test/yq.test.ts @@ -100,7 +100,7 @@ const dir = process.env.BASH_EXPLAINER_YQ; const cmds = [ "yq '.items[] | select(.kind == \"Pod\") | .metadata.name' pods.yml", "yq e '.spec.replicas = 3' d.yml", "yq -i '.version = \"1.2\"' chart.yaml", "yq '.a.b |= . + 1' f.yml", "yq '.users | map(.name)' u.yml", "yq '.tags | join(\",\")' f.yml", "yq '.items | length' f.yml", - "yq -o json '.metadata' f.yml", "yq '.a // \"default\"' f.yml", "yq 'del(.status)' f.yml", "yq '.list | sort | unique' f.yml", + "yq -o json '.metadata' f.yml", "yq -o=json '.metadata' f.yml", "yq '.a // \"default\"' f.yml", "yq 'del(.status)' f.yml", "yq '.list | sort | unique' f.yml", "yq '.items[] | select(.enabled)' f.yml", "yq '.a | has(\"b\")' f.yml", "yq '.items | sort_by(.name) | .[0]' f.yml", "yq '.name | test(\"^web\")' f.yml", "yq '.users[] | select(.age > 30) | .name' f.yml", "yq '[.items[].name]' f.yml", "yq '.a | split(\"-\")' f.yml", "yq '.items | group_by(.kind)' f.yml", "yq -y '.spec | add' x.yml", "yq -y '.users | map(.name)' u.yml", "yq -r '.users[].name' u.yml", "yq -y 'del(.a)' f.yml", "yq -y '.config.timeout // 30' f.yml", @@ -130,3 +130,19 @@ const dir = process.env.BASH_EXPLAINER_YQ; }); } }); + +describe('yq -o as pflag reads it', () => { + // mikefarah/yq reads its flags with pflag, where `-o=json` gives `-o` the value `json` (the `cobra` preset, compared + // with gh in test/parsers-real.test.ts). Its example is the JSON one, as for `-o json`. + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { Analyzer } = require('../src/analyzer'); + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { createUnbashParser } = require('../src/syntax/adapters/unbash'); + it('-o=json, -ojson and -o json give the same example', () => { + const parser = createUnbashParser(); + const example = (line: string) => JSON.stringify(new Analyzer('en').explainLine(parser.parse(line), 0).examples[0]?.sections); + const json = example("yq -o json '.a' f.yml"); + assert.match(json, /\{/); + for (const line of ["yq -o=json '.a' f.yml", "yq -ojson '.a' f.yml", "yq e -o=json '.a' f.yml"]) assert.strictEqual(example(line), json, line); + }); +});