diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc1320f..6c67d03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,21 +1,58 @@ -name: Continuous Integration +name: CI -on: push +on: + pull_request: + push: + branches: + - main jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + env: + MIX_ENV: test + strategy: + fail-fast: false + matrix: + include: + - pair: + elixir: "1.15" + otp: "24" + - pair: + elixir: "1.18" + otp: "27" + - pair: + elixir: "1.20" + otp: "29" + lint: lint steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup Elixir - uses: erlef/setup-beam@v1 + - uses: actions/checkout@v4 + + - uses: erlef/setup-beam@v1 + with: + otp-version: ${{matrix.pair.otp}} + elixir-version: ${{matrix.pair.elixir}} + + - uses: actions/cache@v4 with: - otp-version: '24' - elixir-version: '1.12.1' - - name: Install dependencies - run: mix deps.get - - name: Run tests - run: mix test - - name: Check format - run: mix format --check-formatted \ No newline at end of file + path: | + deps + _build + key: ${{ runner.os }}-mix-${{matrix.pair.elixir}}-${{matrix.pair.otp}}-${{ hashFiles('**/mix.lock') }} + restore-keys: | + ${{ runner.os }}-mix- + + - run: mix deps.get + + - run: mix format --check-formatted + if: ${{ matrix.lint }} + + - run: mix deps.unlock --check-unused + if: ${{ matrix.lint }} + + - run: mix deps.compile + + - run: mix compile --warnings-as-errors + if: ${{ matrix.lint }} + + - run: mix test diff --git a/README.md b/README.md index 48d46bb..dde00db 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Add `makeup_html` to your list of dependencies in `mix.exs`: ```elixir def deps do [ - {:makeup_html, "~> 0.1.0"} + {:makeup_html, "~> 0.2.0"} ] end ``` diff --git a/lib/makeup/lexers/html_lexer.ex b/lib/makeup/lexers/html_lexer.ex index 3af1429..0fe7516 100644 --- a/lib/makeup/lexers/html_lexer.ex +++ b/lib/makeup/lexers/html_lexer.ex @@ -10,117 +10,155 @@ defmodule Makeup.Lexers.HTMLLexer do import Makeup.Lexer.Groups import Makeup.Lexers.HTMLLexer.Combinators - @attributes MapSet.new(get_attributes() ++ get_event_handler_attributes()) - ################################################################### # Step #1: tokenize the input (into a list of tokens) ################################################################### - # Whitespaces - wspace = ascii_string([?\r, ?\s, ?\n, ?\f], min: 1) - whitespace = - wspace + ascii_string([?\r, ?\s, ?\n, ?\t, ?\f], min: 1) |> token(:whitespace) - # Doctype - legacy_doctype_string = - wspace - |> optional() - |> concat(anycase_string("SYSTEM")) - |> optional(wspace) - |> concat( - choice([ - string("\"about:legacy-compat\""), - string("'about:legacy-compat'") - ]) - ) + comment_close = choice([string("-->"), string("--!>")]) + empty_comment = choice([string(">"), string("->")]) + + comment = + string("")) - |> utf8_string([], 1) + quoted_attribute_value = + choice([ + string("\"") |> optional(utf8_string([not: ?"], min: 1)) |> optional(string("\"")), + string("'") |> optional(utf8_string([not: ?'], min: 1)) |> optional(string("'")) + ]) + |> lexeme() + |> token(:string) + + unquoted_attribute_value = + utf8_string([not: ?\s, not: ?\n, not: ?\r, not: ?\t, not: ?\f, not: ?>], min: 1) + |> token(:string) + + # A space may precede a quoted value, not an unquoted one: `field= name="x"` + attribute = + token(attribute_name, :name_attribute) + |> optional( + optional(whitespace) + |> concat(token(string("="), :operator)) + |> optional( + choice([ + optional(whitespace) |> concat(quoted_attribute_value), + unquoted_attribute_value + ]) ) ) - |> string("-->") - |> token(:comment) - - name_tag = - ascii_string([?a..?z, ?A..?Z, ?0..?9, ?_, ?-, ?:, ?.], min: 1) - |> token(:name_tag) - - # Single punctuation symbols - open_tag = - "<" - |> string() - |> token(:punctuation) - |> concat(name_tag) - - close_tag = - ">" - |> string() - |> token(:punctuation) - - close_self_tag = - "/>" - |> string() - |> token(:punctuation) - - open_closing_tag = - " string() - |> token(:punctuation) - |> concat(name_tag) - - # Currently we match attributes anywhere in the text - attributes = utf8_string([?a..?z, ?A..?Z, ?0..?9, ?_, ?-, ?:, ?.], min: 1) |> token(:keyword) - - # Unmatched - insensitive_char = utf8_char([]) |> token(:char) + + # A solidus that does not close the tag, as in `` + stray_solidus = lookahead_not(string("/>")) |> concat(token(string("/"), :punctuation)) + + tag_attributes = repeat(choice([whitespace, attribute, stray_solidus])) + + # An attribute name may contain `<`. A tag with no `>` therefore scans to the + # end of the input. Without `eos` that scan fails and repeats from every `<`. + tag_close = token(choice([string("/>"), string(">")]), :punctuation) + + close_tag = tag("` are ordinary operators in JavaScript and CSS, so a script or + # style body is one opaque token that ends only at its own close tag + raw_element = fn name -> + # `lookahead_not` keeps `` from closing the element with `x` as an attribute. + raw_name = anycase_string(name) |> lookahead_not(ascii_char(tag_name_chars)) + bare_close = token(string(">"), :punctuation) + + open = tag("<", raw_name, tag_attributes, bare_close) + close = tag(" lexeme() + |> token(:text) + + open |> optional(body) |> optional(close) + end + + text = utf8_string([not: ?<, not: ?&], min: 1) |> token(:text) + + any_char = utf8_string([], 1) |> token(:text) # Tag the tokens with the language name. # This makes it easier to postprocess files with multiple languages. @doc false + def __as_html_language__({ttype, meta, value}) when map_size(meta) == 0 do + {ttype, %{language: :html}, value} + end + def __as_html_language__({ttype, meta, value}) do {ttype, Map.put(meta, :language, :html), value} end + # `choice` takes the first match, not the longest, so the order is significant. root_element_combinator = choice([ - # Doctype + # Comments and declarations + comment, + cdata, doctype, - # Operators - operators, - # Delimiters - comment_tag, - open_closing_tag, - open_tag, - close_self_tag, + bogus_comment, + # Tags + raw_element.("script"), + raw_element.("style"), close_tag, + open_tag, # Text - whitespace, - attributes, - insensitive_char + character_reference, + text, + any_char ]) ############################################################################## @@ -148,135 +186,8 @@ defmodule Makeup.Lexers.HTMLLexer do # Step #2: postprocess the list of tokens ################################################################### - # Converts traces of the form [char]+ into a single string - # Converts keywords before and after strings into a single string - - defp merge([{:char, attr, value} | tokens]), - do: merge([{:string, attr, <>} | tokens]) - - defp merge([{tag, attr, value1}, {:char, _attr, value2} | tokens]) - when tag in [:keyword, :string], - do: merge([{:string, attr, <>} | tokens]) - - defp merge([{tag, attr, value1}, {:keyword, _attr, value2} | tokens]) - when tag in [:keyword, :string], - do: merge([{:string, attr, value1 <> value2} | tokens]) - - defp merge([{tag, attr, value1}, {:string, _attr, value2} | tokens]) - when tag in [:keyword, :string], - do: merge([{:string, attr, value1 <> value2} | tokens]) - - defp merge([token | tokens]), - do: [token | merge(tokens)] - - defp merge([]), do: [] - - # Converts the proper keywords into attributes - - defp attributify( - [ - {:punctuation, _, "<"} = punctuation, - {:name_tag, _, _} = name_tag, - {:whitespace, _, _} = whitespace | tokens - ], - _ - ) do - [punctuation, name_tag, whitespace | attributify(tokens, true)] - end - - defp attributify([{:punctuation, _, ">"} = punctuation | tokens], _flag), - do: [punctuation | attributify(tokens, false)] - - defp attributify([{:punctuation, _, "/>"} = punctuation | tokens], _flag), - do: [punctuation | attributify(tokens, false)] - - # when using the HTMLLexer from HEEx, we often have to deal with - # strings like
...
where an attribute starts, but is missing - # a value. We special case this "missing attribute value" case here to avoid - # formatting the closing tag as an attribute. - defp attributify( - [ - {:keyword, attr, value}, - {:operator, _, _} = operator, - {:punctuation, attr2, value2} | tokens - ], - true - ) do - [ - {:name_attribute, attr, value}, - operator, - {:punctuation, attr2, value2} - | attributify(tokens, false) - ] - end - - defp attributify( - [ - {:keyword, attr, value}, - {:operator, _, _} = operator, - {_, attr2, value2} | tokens - ], - true - ) do - [ - {:name_attribute, attr, value}, - operator, - {:string, attr2, value2} - | attributify(tokens, true) - ] - end - - defp attributify([{:keyword, attr, value} | tokens], true), - do: [{:name_attribute, attr, value} | attributify(tokens, true)] - - defp attributify([{:keyword, attr, value} | tokens], flag) do - attribute = - if Enum.member?(@attributes, value), - do: {:name_attribute, attr, value}, - else: {:string, attr, value} - - [attribute | attributify(tokens, flag)] - end - - defp attributify([token | tokens], flag), - do: [token | attributify(tokens, flag)] - - defp attributify([], _), do: [] - - # Converts the content of an element into a string - - defp buffer_to_acc("", acc), do: acc - defp buffer_to_acc(string, acc), do: [{:string, %{language: :html}, string} | acc] - - defp stringify([{:punctuation, _, ">"} = punctuation | tokens], _, buffer), - do: buffer_to_acc(buffer, [punctuation | stringify(tokens, true, "")]) - - # We respect the comments - defp stringify([{:comment, _, _} = comment | tokens], _, buffer), - do: buffer_to_acc(buffer, [comment | stringify(tokens, true, "")]) - - defp stringify([{:punctuation, _, " value) - - defp stringify([], _, buffer), - do: buffer_to_acc(buffer, []) - @impl Makeup.Lexer - def postprocess(tokens, _opts \\ []) do - tokens - |> merge() - |> attributify(false) - |> stringify(false, "") - end + def postprocess(tokens, _opts \\ []), do: tokens ####################################################################### # Step #3: highlight matching delimiters diff --git a/lib/makeup/lexers/html_lexer/application.ex b/lib/makeup/lexers/html_lexer/application.ex index 51ba28b..1501ff9 100644 --- a/lib/makeup/lexers/html_lexer/application.ex +++ b/lib/makeup/lexers/html_lexer/application.ex @@ -9,7 +9,7 @@ defmodule Makeup.Lexers.HTMLLexer.Application do Registry.register_lexer(HTMLLexer, options: [], names: ["html"], - extensions: ["html"] + extensions: ["html", "htm", "xhtml"] ) Supervisor.start_link([], strategy: :one_for_one) diff --git a/lib/makeup/lexers/html_lexer/combinators.ex b/lib/makeup/lexers/html_lexer/combinators.ex index e9e4452..c06a0a9 100644 --- a/lib/makeup/lexers/html_lexer/combinators.ex +++ b/lib/makeup/lexers/html_lexer/combinators.ex @@ -3,7 +3,22 @@ defmodule Makeup.Lexers.HTMLLexer.Combinators do import NimbleParsec import Makeup.Lexer.Combinators - def keyword(string), do: string |> token(:keyword) + def tag(opening, name, attributes, closing) do + token(string(opening), :punctuation) + |> concat(token(name, :name_tag)) + |> concat(attributes) + |> concat(closing) + end + + # The terminator is left unconsumed. + def chars_until(terminator, opts \\ []) do + character = lookahead_not(terminator) |> utf8_string([], 1) + + case Keyword.get(opts, :min, 0) do + 0 -> repeat(character) + min -> times(character, min: min) + end + end # Insensitive string # https://elixirforum.com/t/nimbleparsec-case-insensitive-matches/14339/2 @@ -41,221 +56,4 @@ defmodule Makeup.Lexers.HTMLLexer.Combinators do defp both_cases(c) do [c, c + 32] end - - def get_attributes do - [ - "abbr", - "accept", - "accept-charset", - "accesskey", - "action", - "allow", - "allowfullscreen", - "allowpaymentrequest", - "alt", - "as", - "async", - "autocapitalize", - "autocomplete", - "autofocus", - "autoplay", - "charset", - "checked", - "cite", - "class", - "color", - "cols", - "colspan", - "content", - "contenteditable", - "controls", - "coords", - "crossorigin", - "data", - "datetime", - "decoding", - "default", - "defer", - "dir", - "dirname", - "disabled", - "download", - "draggable", - "enctype", - "enterkeyhint", - "for", - "form", - "formaction", - "formenctype", - "formmethod", - "formnovalidate", - "formtarget", - "headers", - "height", - "hidden", - "high", - "href", - "hreflang", - "http-equiv", - "id", - "imagesizes", - "imagesrcset", - "inputmode", - "integrity", - "is", - "ismap", - "itemid", - "itemprop", - "itemref", - "itemscope", - "itemtype", - "kind", - "label", - "lang", - "list", - "loading", - "loop", - "low", - "manifest", - "max", - "maxlength", - "media", - "method", - "min", - "minlength", - "multiple", - "muted", - "name", - "nomodule", - "nonce", - "novalidate", - "open", - "optimum", - "pattern", - "ping", - "placeholder", - "playsinline", - "poster", - "preload", - "readonly", - "referrerpolicy", - "rel", - "required", - "reversed", - "rows", - "rowspan", - "sandbox", - "scope", - "selected", - "shape", - "size", - "sizes", - "slot", - "span", - "spellcheck", - "src", - "srcdoc", - "srclang", - "srcset", - "start", - "step", - "style", - "tabindex", - "target", - "title", - "translate", - "type", - "usemap", - "value", - "width", - "wrap" - ] - end - - def get_event_handler_attributes do - [ - "onabort", - "onafterprint", - "onauxclick", - "onbeforeprint", - "onbeforeunload", - "onblur", - "oncancel", - "oncanplay", - "oncanplaythrough", - "onchange", - "onclick", - "onclose", - "oncontextmenu", - "oncopy", - "oncuechange", - "oncut", - "ondblclick", - "ondrag", - "ondragend", - "ondragenter", - "ondragexit", - "ondragleave", - "ondragover", - "ondragstart", - "ondrop", - "ondurationchange", - "onemptied", - "onended", - "onerror", - "onfocus", - "onformdata", - "onhashchange", - "oninput", - "oninvalid", - "onkeydown", - "onkeypress", - "onkeyup", - "onlanguagechange", - "onload", - "onloadeddata", - "onloadedmetadata", - "onloadstart", - "onmessage", - "onmessageerror", - "onmousedown", - "onmouseenter", - "onmouseleave", - "onmousemove", - "onmouseout", - "onmouseover", - "onmouseup", - "onoffline", - "ononline", - "onpagehide", - "onpageshow", - "onpaste", - "onpause", - "onplay", - "onplaying", - "onpopstate", - "onprogress", - "onratechange", - "onrejectionhandled", - "onreset", - "onresize", - "onscroll", - "onsecuritypolicyviolation", - "onseeked", - "onseeking", - "onselect", - "onslotchange", - "onstalled", - "onstorage", - "onsubmit", - "onsuspend", - "ontimeupdate", - "ontoggle", - "onunhandledrejection", - "onunload", - "onvolumechange", - "onwaiting", - "onwheel" - ] - end end diff --git a/mix.exs b/mix.exs index 8aeceee..1198e99 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule MakeupHTML.Mixfile do [ app: :makeup_html, version: "0.2.0", - elixir: "~> 1.10", + elixir: "~> 1.12", start_permanent: Mix.env() == :prod, deps: deps(), elixirc_paths: elixirc_paths(Mix.env()), @@ -49,7 +49,7 @@ defmodule MakeupHTML.Mixfile do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:makeup, "~> 1.2"}, + {:makeup, "~> 1.2.3 or ~> 1.3"}, {:stream_data, "~> 1.0", only: :test}, {:ex_doc, "~> 0.24", only: :dev, runtime: false} ] diff --git a/mix.lock b/mix.lock index 4bd2cae..eace716 100644 --- a/mix.lock +++ b/mix.lock @@ -1,9 +1,9 @@ %{ "earmark_parser": {:hex, :earmark_parser, "1.4.43", "34b2f401fe473080e39ff2b90feb8ddfeef7639f8ee0bbf71bb41911831d77c5", [:mix], [], "hexpm", "970a3cd19503f5e8e527a190662be2cee5d98eed1ff72ed9b3d1a3d466692de8"}, "ex_doc": {:hex, :ex_doc, "0.37.0", "970f92b39e62c460aa8a367508e938f5e4da6e2ff3eaed3f8530b25870f45471", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "b0ee7f17373948e0cf471e59c3a0ee42f3bd1171c67d91eb3626456ef9c6202c"}, - "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, + "makeup": {:hex, :makeup, "1.2.3", "c3ef0ff83acf305384acba0f70bff87d8732502baa1eb3eb081d8c154fbe18f5", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "15970751b7021c2bcd7ed56a54b3def23affd3969fb7c214dffdf28e97ce6f76"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, - "stream_data": {:hex, :stream_data, "1.1.2", "05499eaec0443349ff877aaabc6e194e82bda6799b9ce6aaa1aadac15a9fdb4d", [:mix], [], "hexpm", "129558d2c77cbc1eb2f4747acbbea79e181a5da51108457000020a906813a1a9"}, + "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, } diff --git a/test/makeup/html_lexer/html_lexer_property_test.exs b/test/makeup/html_lexer/html_lexer_property_test.exs new file mode 100644 index 0000000..2d1b869 --- /dev/null +++ b/test/makeup/html_lexer/html_lexer_property_test.exs @@ -0,0 +1,28 @@ +defmodule HTMLLexerProperty do + use ExUnit.Case, async: true + use ExUnitProperties + + alias Makeup.Formatters.HTML.HTMLFormatter + alias Makeup.Lexer + alias Makeup.Lexers.HTMLLexer + + property "lexing is lossless" do + check all(source <- HTMLGenerators.document(), max_runs: 2_000) do + assert source |> HTMLLexer.lex() |> Lexer.unlex() == source + end + end + + property "every token value is a binary" do + check all(source <- HTMLGenerators.document(), max_runs: 2_000) do + for {_ttype, _meta, value} <- HTMLLexer.lex(source) do + assert is_binary(value) + end + end + end + + property "the formatter renders every token stream" do + check all(source <- HTMLGenerators.document(), max_runs: 2_000) do + assert is_binary(HTMLFormatter.format_as_binary(HTMLLexer.lex(source))) + end + end +end diff --git a/test/makeup/html_lexer/html_lexer_tokenizer_test.exs b/test/makeup/html_lexer/html_lexer_tokenizer_test.exs index a1b7525..221a760 100644 --- a/test/makeup/html_lexer/html_lexer_tokenizer_test.exs +++ b/test/makeup/html_lexer/html_lexer_tokenizer_test.exs @@ -1,10 +1,8 @@ defmodule HTMLLexerTokenizer do - use ExUnit.Case, async: false - use ExUnitProperties + use ExUnit.Case, async: true alias Makeup.Lexers.HTMLLexer alias Makeup.Lexer.Postprocess - alias Helper # This function has three purposes: # 1. Ensure deterministic lexer output (no random prefix) @@ -19,45 +17,6 @@ defmodule HTMLLexerTokenizer do |> Enum.map(fn {ttype, meta, value} -> {ttype, Map.delete(meta, :language), value} end) end - defp tokenize_elements(element) do - [element_name | attributes] = - element - |> String.split(" ") - - attributes_tokens = - attributes - |> Enum.flat_map(fn attr -> - [name | value] = - attr - |> String.split("=") - - cond do - value != [] -> - [ - {:whitespace, %{}, " "}, - {:name_attribute, %{}, name}, - {:operator, %{}, "="}, - {:string, %{}, hd(value)} - ] - - String.length(name) == 0 -> - [ - {:whitespace, %{}, " "} - ] - - true -> - [ - {:whitespace, %{}, " "}, - {:name_attribute, %{}, name} - ] - end - end) - - if attributes_tokens == [], - do: {{:name_tag, %{}, element_name}, {:whitespace, %{}, " "}}, - else: {{:name_tag, %{}, element_name}, attributes_tokens} - end - ################################################################### # Empty string ################################################################### @@ -69,18 +28,6 @@ defmodule HTMLLexerTokenizer do # Doctype ################################################################### describe "DOCTYPE" do - property "correct DOCTYPE is correctly tokenized" do - check all(doctype <- HTMLGenerators.doctype()) do - assert lex(doctype) == [{:comment_preproc, %{}, doctype}] - end - end - - property "incorrect DOCTYPE is incorrectly tokenized" do - check all(doctype <- HTMLGenerators.incorrect_doctype()) do - refute lex(doctype) == [{:comment_preproc, %{}, doctype}] - end - end - test "" do doctype = "" @@ -100,53 +47,167 @@ defmodule HTMLLexerTokenizer do # Comment ################################################################### describe "comment" do - property "correct comment is correctly tokenized" do - check all(comment <- HTMLGenerators.comment()) do - assert lex(comment) == [{:comment, %{}, comment}] - end - end - test "" do comment = "" - assert lex(comment) == [{:comment, %{}, ""}] + assert lex(comment) == [ + {:comment_multiline, %{}, ""} + ] + end + + test "an empty comment closes on >" do + assert lex("x") == [{:comment_multiline, %{}, ""}, {:text, %{}, "x"}] + end + + test "an empty comment closes on ->" do + assert lex("x") == [{:comment_multiline, %{}, ""}, {:text, %{}, "x"}] + end + + test " is a comment" do + assert lex("") == [{:comment_multiline, %{}, ""}] + end + + test "--!> closes the comment" do + assert lex("

y

") == [ + {:comment_multiline, %{}, ""}, + {:punctuation, %{group_id: "group-1"}, "<"}, + {:name_tag, %{}, "p"}, + {:punctuation, %{group_id: "group-1"}, ">"}, + {:text, %{}, "y"}, + {:punctuation, %{group_id: "group-2"}, ""} + ] + end + + test "--!> closes the comment before a later -->" do + assert lex(" b -->") == [ + {:comment_multiline, %{}, ""}, + {:text, %{}, " b -->"} + ] + end + + test "an unterminated comment runs to the end of the input" do + assert lex(""}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, + {:comment_multiline, %{}, ""}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-2"}, "<"}, {:name_tag, %{}, "head"}, {:punctuation, %{group_id: "group-2"}, ">"}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-3"}, "<"}, {:name_tag, %{}, "title"}, {:punctuation, %{group_id: "group-3"}, ">"}, - {:string, %{}, "\n Hello\n "}, + {:text, %{}, " + Hello + "}, {:punctuation, %{group_id: "group-4"}, ""}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-5"}, ""}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-6"}, "<"}, {:name_tag, %{}, "body"}, {:punctuation, %{group_id: "group-6"}, ">"}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-7"}, "<"}, {:name_tag, %{}, "p"}, {:punctuation, %{group_id: "group-7"}, ">"}, - {:string, %{}, "\n Welcome to this example.\n "}, + {:text, %{}, " + Welcome to this example. + "}, {:punctuation, %{group_id: "group-8"}, ""}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-9"}, ""}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-10"}, ""}, - {:string, %{}, "\n"} + {:text, %{}, " +"} ] end end @@ -501,11 +680,12 @@ defmodule HTMLLexerTokenizer do {:whitespace, %{}, " "}, {:name_attribute, %{}, ":let"}, {:operator, %{}, "="}, - {:string, %{}, " "}, + {:whitespace, %{}, " "}, {:name_attribute, %{}, "field"}, {:operator, %{}, "="}, {:punctuation, %{group_id: "group-1"}, ">"}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-2"}, "<"}, {:name_tag, %{}, "input"}, {:whitespace, %{}, " "}, @@ -519,9 +699,10 @@ defmodule HTMLLexerTokenizer do {:whitespace, %{}, " "}, {:name_attribute, %{}, "value"}, {:operator, %{}, "="}, - {:string, %{}, " "}, + {:whitespace, %{}, " "}, {:punctuation, %{group_id: "group-2"}, "/>"}, - {:whitespace, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-3"}, "<"}, {:name_tag, %{}, ".input"}, {:whitespace, %{}, " "}, @@ -531,13 +712,14 @@ defmodule HTMLLexerTokenizer do {:whitespace, %{}, " "}, {:name_attribute, %{}, "field"}, {:operator, %{}, "="}, - {:string, %{}, " "}, + {:whitespace, %{}, " "}, {:name_attribute, %{}, "placeholder"}, {:operator, %{}, "="}, {:string, %{}, "\"email\""}, {:whitespace, %{}, " "}, {:punctuation, %{group_id: "group-3"}, "/>"}, - {:whitespace, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-4"}, "<"}, {:name_tag, %{}, ".input"}, {:whitespace, %{}, " "}, @@ -547,17 +729,19 @@ defmodule HTMLLexerTokenizer do {:whitespace, %{}, " "}, {:name_attribute, %{}, "field"}, {:operator, %{}, "="}, - {:string, %{}, " "}, + {:whitespace, %{}, " "}, {:name_attribute, %{}, "placeholder"}, {:operator, %{}, "="}, {:string, %{}, "\"name\""}, {:whitespace, %{}, " "}, {:punctuation, %{group_id: "group-4"}, "/>"}, - {:whitespace, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-5"}, "<"}, {:name_tag, %{}, "label"}, {:punctuation, %{group_id: "group-5"}, ">"}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-6"}, "<"}, {:name_tag, %{}, "input"}, {:whitespace, %{}, " "}, @@ -571,33 +755,35 @@ defmodule HTMLLexerTokenizer do {:whitespace, %{}, " "}, {:name_attribute, %{}, "value"}, {:operator, %{}, "="}, - {:string, %{}, " "}, + {:whitespace, %{}, " "}, {:name_attribute, %{}, "class"}, {:operator, %{}, "="}, {:string, %{}, "\"hidden\""}, {:whitespace, %{}, " "}, {:punctuation, %{group_id: "group-6"}, "/>"}, - {:whitespace, %{}, "\n "}, - {:string, %{}, "delete"}, - {:whitespace, %{}, "\n "}, + {:text, %{}, " + delete + "}, {:punctuation, %{group_id: "group-7"}, ""}, - {:string, %{}, "\n"}, + {:text, %{}, " +"}, {:punctuation, %{group_id: "group-8"}, ""}, - {:string, %{}, "\n\n"}, + {:text, %{}, " + +"}, {:punctuation, %{group_id: "group-9"}, "<"}, {:name_tag, %{}, "label"}, {:whitespace, %{}, " "}, {:name_attribute, %{}, "class"}, {:operator, %{}, "="}, - {:string, %{}, "\"block"}, - {:whitespace, %{}, " "}, - {:string, %{}, "cursor-pointer\""}, + {:string, %{}, "\"block cursor-pointer\""}, {:punctuation, %{group_id: "group-9"}, ">"}, - {:string, %{}, "\n "}, + {:text, %{}, " + "}, {:punctuation, %{group_id: "group-10"}, "<"}, {:name_tag, %{}, "input"}, {:whitespace, %{}, " "}, @@ -614,15 +800,14 @@ defmodule HTMLLexerTokenizer do {:string, %{}, "\"hidden\""}, {:whitespace, %{}, " "}, {:punctuation, %{group_id: "group-10"}, "/>"}, - {:whitespace, %{}, "\n "}, - {:string, %{}, "add"}, - {:whitespace, %{}, " "}, - {:string, %{}, "more"}, - {:whitespace, %{}, "\n"}, + {:text, %{}, " + add more +"}, {:punctuation, %{group_id: "group-11"}, ""}, - {:string, %{}, "\n"} + {:text, %{}, " +"} ] = lex(element) end end diff --git a/test/makeup/html_lexer/registry_test.exs b/test/makeup/html_lexer/registry_test.exs index 209203f..71cb701 100644 --- a/test/makeup/html_lexer/registry_test.exs +++ b/test/makeup/html_lexer/registry_test.exs @@ -9,8 +9,10 @@ defmodule Makeup.Lexers.HTMLLexer.RegistryTest do assert {:ok, {HTMLLexer, []}} == Registry.fetch_lexer_by_name("html") end - test "file extension" do - assert {:ok, {HTMLLexer, []}} == Registry.fetch_lexer_by_extension("html") + test "file extensions" do + for extension <- ["html", "htm", "xhtml"] do + assert {:ok, {HTMLLexer, []}} == Registry.fetch_lexer_by_extension(extension) + end end end end diff --git a/test/support/helper.ex b/test/support/helper.ex deleted file mode 100644 index 9863be9..0000000 --- a/test/support/helper.ex +++ /dev/null @@ -1,19 +0,0 @@ -defmodule Helper do - @moduledoc false - - def insensitive_case_string(string) do - string - |> String.split("", trim: true) - |> insensitive_case_string([]) - |> Enum.join("") - end - - def insensitive_case_string([], result), do: result - - def insensitive_case_string([h | t], result) do - insensitive_case_string(t, result ++ [insensitive_fun(Enum.random(0..1), h)]) - end - - defp insensitive_fun(0, string), do: String.downcase(string) - defp insensitive_fun(1, string), do: String.upcase(string) -end diff --git a/test/support/html_generators.ex b/test/support/html_generators.ex index 3be44be..a7b668b 100644 --- a/test/support/html_generators.ex +++ b/test/support/html_generators.ex @@ -2,341 +2,160 @@ defmodule HTMLGenerators do @moduledoc false use ExUnitProperties - alias Makeup.Lexers.HTMLLexer.Combinators - - @attributes Combinators.get_attributes() ++ Combinators.get_event_handler_attributes() - - defp get_attributes do - @attributes - end - - defp get_elements do - [ - "a", - "abbr", - "address", - "area", - "article", - "aside", - "audio", - "b", - "base", - "bdi", - "bdo", - "blockquote", - "body", - "br", - "button", - "canvas", - "caption", - "cite", - "code", - "col", - "colgroup", - "data", - "datalist", - "dd", - "del", - "details", - "dfn", - "dialog", - "div", - "dl", - "dt", - "em", - "embed", - "fieldset", - "figcaption", - "figure", - "footer", - "form", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "head", - "header", - "hgroup", - "hr", - "html", - "i", - "iframe", - "img", - "input", - "ins", - "kbd", - "label", - "legend", - "li", - "link", - "main", - "map", - "mark", - "math", - "menu", - "meta", - "meter", - "nav", - "noscript", - "object", - "ol", - "optgroup", - "option", - "output", - "p", - "param", - "picture", - "pre", - "progress", - "q", - "rp", - "rt", - "ruby", - "s", - "samp", - "script", - "section", - "select", - "slot", - "small", - "source", - "span", - "strong", - "style", - "sub", - "summary", - "sup", - "svg", - "table", - "tbody", - "td", - "template", - "textarea", - "tfoot", - "th", - "thead", - "time", - "title", - "tr", - "track", - "u", - "ul", - "var", - "video", - "wbr" - ] + def tag_name do + one_of([ + gen all( + first <- string([?a..?z, ?A..?Z, ?., ?:], length: 1), + rest <- string([?a..?z, ?A..?Z, ?0..?9, ?-, ?_, ?:, ?.], max_length: 5) + ) do + first <> rest + end, + member_of(["div", "DIV", "svg:rect", ".input", ":inner_block", "my-element", "_x"]) + ]) end - defp insensitive_case_string(string) do - string - |> String.split("", trim: true) - |> insensitive_case_string([]) - |> Enum.join("") + def attribute_name do + string([?a..?z, ?A..?Z, ?0..?9, ?-, ?_, ?:, ?@, ?., ?#, ?<, ?{, ?}], + min_length: 1, + max_length: 8 + ) end - defp insensitive_case_string([], result), do: result - - defp insensitive_case_string([h | t], result) do - insensitive_case_string(t, result ++ [insensitive_fun(Enum.random(0..1), h)]) - end - - defp insensitive_fun(0, string), do: String.downcase(string) - defp insensitive_fun(1, string), do: String.upcase(string) - - ## Generators - - def doctype_legacy_string do - ExUnitProperties.gen all( - one_or_more <- StreamData.integer(1..5), - quotation <- StreamData.member_of(["\"", "\'"]) - ) do - String.duplicate(" ", one_or_more) <> - insensitive_case_string("SYSTEM") <> - String.duplicate(" ", one_or_more) <> quotation <> "about:legacy-compat" <> quotation - end - end - - def doctype do - ExUnitProperties.gen all( - one_or_more <- StreamData.integer(1..5), - optional <- StreamData.integer(0..5), - legacy_string <- doctype_legacy_string() - ) do - " - insensitive_case_string("DOCTYPE") <> - String.duplicate(" ", one_or_more) <> - insensitive_case_string("html") <> - legacy_string <> - String.duplicate(" ", optional) <> ">" + def attribute do + gen all( + name <- attribute_name(), + quotation <- member_of(["\"", "'", ""]), + value <- string(:alphanumeric, max_length: 6), + before_equals <- member_of(["", " "]), + after_equals <- member_of(["", " "]), + valued? <- boolean() + ) do + if valued?, + do: name <> before_equals <> "=" <> after_equals <> quotation <> value <> quotation, + else: name end end - def comment do - ExUnitProperties.gen all(gen_text <- StreamData.string(:ascii)) do - text = - gen_text - |> String.replace_leading(">", "") - |> String.replace_leading("->", "") - |> String.replace("", "") - |> String.replace("--!>", "") - |> String.replace_trailing(" text <> "-->" + def attributes do + gen all(attributes <- list_of(attribute(), max_length: 3)) do + Enum.map_join(attributes, &(" " <> &1)) end end - def void_element do - ExUnitProperties.gen all(element <- StreamData.member_of(get_elements())) do - "<" <> element <> ">" + def start_tag do + gen all( + name <- tag_name(), + attributes <- attributes(), + closing <- member_of([">", "/>"]) + ) do + "<" <> name <> attributes <> closing end end - def attribute do - ExUnitProperties.gen all( - quotation <- StreamData.member_of(["\"", "\'", ""]), - name <- StreamData.member_of(get_attributes()), - value <- StreamData.string(:alphanumeric), - value not in get_attributes() - ) do - if String.length(value) != 0, - do: name <> "=" <> quotation <> value <> quotation, - else: name + def end_tag do + gen all( + name <- tag_name(), + extra <- member_of(["", " ", " extra", " a=b"]) + ) do + " name <> extra <> ">" end end - def element_attribute do - ExUnitProperties.gen all( - quotation <- StreamData.member_of(["\"", "\'", ""]), - name <- StreamData.member_of(get_attributes()), - value <- StreamData.string(:alphanumeric), - value not in get_attributes() - ) do - if String.length(value) != 0, - do: name <> "=" <> quotation <> value <> quotation, - else: name + def comment do + gen all( + body <- string([?a..?z, ?\s, ?<, ?>, ?!, ?-], max_length: 12), + closing <- member_of(["-->", "--!>"]) + ) do + "", "", ""]) end - def nested_element do - ExUnitProperties.gen all( - element_name <- StreamData.member_of(get_elements()), - content <- StreamData.one_of([void_element(), single_element()]), - attributes <- StreamData.list_of(element_attribute(), max_length: 3) - ) do - "<" <> - element_name <> - " " <> - Enum.join(attributes, " ") <> - ">" <> content <> " element_name <> ">" + def cdata do + gen all( + opening <- member_of([", ?]], max_length: 12) + ) do + opening <> body <> "]]>" end end - def element do - ExUnitProperties.gen all( - element <- - StreamData.one_of([ - void_element(), - single_element(), - nested_element() - ]) - ) do - element + def doctype do + gen all( + keyword <- member_of(["DOCTYPE", "doctype", "DocType"]), + rest <- string([?a..?z, ?\s], max_length: 8) + ) do + " keyword <> " " <> rest <> ">" end end - def document do - ExUnitProperties.gen all( - bom <- StreamData.member_of([:unicode.encoding_to_bom(:utf8), ""]), - any_comments <- StreamData.list_of(comment(), max_length: 3), - doctype <- doctype(), - document_content <- StreamData.list_of(element(), max_length: 3) - ) do - comments = Enum.join(any_comments, "\n") - document = "\n" <> Enum.join(document_content, "\n") <> "\n" - - bom <> - comments <> "\n" <> doctype <> "\n" <> comments <> "\n" <> document <> "\n" <> comments + def bogus_comment do + gen all( + opening <- member_of([" body <> ">" end end - ################################################################### - # Incorrect - ################################################################### - - def incorrect_doctype do - ExUnitProperties.gen all(doctype <- doctype()) do - doctype - |> String.replace_suffix(">", "") + def character_reference do + gen all( + reference <- + one_of([ + string([?a..?z], min_length: 2, max_length: 5), + map(string([?0..?9], min_length: 1, max_length: 4), &("#" <> &1)) + ]) + ) do + "&" <> reference <> ";" end end - def incorrect_void_element do - ExUnitProperties.gen all(void_element <- void_element()) do - void_element - |> String.replace_suffix(">", "") + def raw_element do + gen all( + name <- member_of(["script", "style", "SCRIPT", "Style"]), + attributes <- attributes(), + body <- string([?a..?z, ?<, ?>, ?=, ?/, ?!, ?\s], max_length: 14) + ) do + "<" <> name <> attributes <> ">" <> body <> " name <> ">" end end - def incorrect_single_element do - ExUnitProperties.gen all(single_element <- single_element()) do - single_element - |> String.replace_suffix(">", "") - end + def text do + string([?a..?z, ?A..?Z, ?0..?9, ?\s, ?\n, ?&, ?<, ?>, ?., ?-, ?!, ?é, ?€], max_length: 12) end - def incorrect_nested_element do - ExUnitProperties.gen all(nested_element <- nested_element()) do - nested_element - |> String.replace_suffix(">", "") - end + # Drops the last character, so `` becomes `