refactor the world

big simplification/XHTML support

Author: Porges

Cargo.lock

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = [ "html2rdf" , "html2rdf-cli"]
+members = [ "html2rdf", "html2rdf-cli", "librdfa-wrapper" ]
 
 [workspace.package]
 authors = ["George Pollard <porges@porg.es>"]
@@ -13,10 +13,22 @@ repository = "https://github.com/Porges/html2rdf"
 rust-version = "1.88.0"
 
 [workspace.dependencies]
+derive_more = { version = "^2.1.1", features = ["display", "error"] }
 html2rdf = { path = "html2rdf" }
+icu_locale =  "2.1.1"
+jiff = "^0.2.23"
+librdfa-wrapper = { path = "librdfa-wrapper" }
 oxiri = "0.2.11"
 oxrdf = "0.3.3"
 oxttl = "0.2.3"
+reqwest = { version = "^0.13.2", features = ["blocking", "charset", "http2", "native-tls", "system-proxy"] }
+
+[profile.release]
+debug = 1
+
+[profile.profiling]
+inherits = "release"
+debug = true
 
 [profile.fuzz]
 inherits = "dev"

Cargo.toml

@@ -16,12 +16,14 @@ path = "src/main.rs"
 
 [dependencies]
 clap = { version = "4.5.59", features = ["cargo", "derive", "wrap_help"] }
+derive_more = { workspace = true, features = ["from", "from_str"] } #unified
+dunce = "1.0.5"
 html2rdf = { workspace = true }
-jiff = "0.2.20"
+jiff = { workspace = true } #unified
 oxiri = { workspace = true }
 oxrdf = { workspace = true, features = ["rdfc-10"] }
 oxttl = { workspace = true }
-reqwest = { version = "0.13.2", default-features = false, features = ["charset", "http2", "native-tls", "system-proxy"] }
+reqwest = { workspace = true, features = ["blocking"] } #unified
 tokio = { version = "1.49.0", features = ["macros", "rt"] }
 tracing-subscriber = { version = "0.3.22" }
 url = "2.5.8"

html2rdf-cli/Cargo.toml

@@ -1,57 +1,99 @@
 use std::{io::Write, process::ExitCode};
 
 use clap::Parser;
-use oxrdf::graph::{CanonicalizationAlgorithm, CanonicalizationHashAlgorithm};
+use html2rdf::{Options, algorithms::OnlineVocabularyResolver, host_language::Html5};
+use oxrdf::{
+    Graph,
+    graph::{CanonicalizationAlgorithm, CanonicalizationHashAlgorithm},
+};
 
 #[derive(Parser)]
 #[command(version, about)]
 struct Args {
-    #[arg(value_name = "URL")]
-    target: url::Url,
+    #[clap(flatten)]
+    input: InputGroup,
 
+    /// Canonicalize the resulting graph.
     #[arg(long, short = 'c')]
     canonicalize: bool,
+
+    /// Perform RDFa vocabulary expansion.
+    #[arg(long)]
+    vocab_expansion: bool,
+}
+
+#[derive(Debug, clap::Args)]
+#[group(required = true, multiple = false)]
+pub struct InputGroup {
+    /// The target URL (must be absolute).
+    #[arg(value_name = "URL")]
+    url: Option<url::Url>,
+
+    /// The target path (can be relative).
+    #[arg(long, value_name = "PATH")]
+    path: Option<std::path::PathBuf>,
+}
+
+#[derive(derive_more::Error, derive_more::Display, Debug)]
+#[display("Unsupported URL scheme `{scheme}`.")]
+struct UnsupportedUrlScheme {
+    scheme: String,
 }
 
 #[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<ExitCode, Box<dyn std::error::Error>> {
     tracing_subscriber::fmt::init();
 
     let args = Args::parse();
-    let client = reqwest::Client::new();
-    let base = args.target.to_string();
-    let base_iri = oxiri::Iri::parse(base.clone())?;
-    let response = client.get(args.target).send().await?.error_for_status()?;
-    let content_type = response
-        .headers()
-        .get(reqwest::header::CONTENT_TYPE)
-        .and_then(|v| v.to_str().ok());
-
-    if content_type.is_some_and(|ct| !ct.starts_with("text/html")) {
-        eprintln!("Error: content type is not text/html.");
-        return Ok(ExitCode::FAILURE);
-    }
-
-    drop(client);
-
-    let final_url = response.url().clone();
+    let base = if let Some(url) = args.input.url {
+        url
+    } else {
+        let p = args.input.path.unwrap(); // UNWRAP: clap guarantee
+        url::Url::from_file_path(dunce::canonicalize(p)?).unwrap()
+    };
+    let base_iri = oxiri::Iri::parse(base.as_str())?;
+
+    let mut final_url = base.clone(); // the final, resolved, URL
+    let content: String = match base.scheme() {
+        "http" | "https" => {
+            let client = reqwest::Client::new();
+            let response = client.get(base.clone()).send().await?.error_for_status()?;
+            let content_type = response
+                .headers()
+                .get(reqwest::header::CONTENT_TYPE)
+                .and_then(|v| v.to_str().ok());
+
+            if content_type.is_some_and(|ct| !ct.starts_with("text/html")) {
+                eprintln!("Error: content type is not text/html.");
+                return Ok(ExitCode::FAILURE);
+            }
+
+            final_url = response.url().clone();
+            response.text().await?
+        }
+        "file" => std::fs::read_to_string(base.to_file_path().unwrap())?,
+        scheme => {
+            return Err(UnsupportedUrlScheme {
+                scheme: scheme.to_string(),
+            }
+            .into());
+        }
+    };
 
-    let content = response.text().await?;
-    let mut output_graph = oxrdf::Graph::new();
-    let mut processor_graph = oxrdf::Graph::new();
-    html2rdf::process(
-        &content,
-        base_iri.clone(),
-        &mut output_graph,
-        &mut processor_graph,
-    )?;
+    let mut options = Options::<Html5>::default();
+    if args.vocab_expansion {
+        options = options.enable_vocabulary_expansion(OnlineVocabularyResolver::default());
+    }
+    let (mut output_graph, processor_graph) =
+        html2rdf::doc_to_graphs::<_, Graph>(&content, base_iri.as_ref(), options)
+            .unwrap_or_else(|inf| match inf {});
 
     {
         // output any warnings/errors
         let serializer = oxttl::TurtleSerializer::new();
         let mut locked_err = std::io::stderr().lock();
         let mut writer = serializer.for_writer(&mut locked_err);
-        for triple in processor_graph.iter() {
+        for triple in &processor_graph {
             writer.serialize_triple(triple)?;
         }
 
@@ -61,8 +103,8 @@ async fn main() -> Result<ExitCode, Box<dyn std::error::Error>> {
 
     {
         // use serializer with all known prefixes
-        let serializer = html2rdf::initial_context_prefixes().mappings().try_fold(
-            oxttl::TurtleSerializer::new().with_base_iri(base)?,
+        let serializer = html2rdf::initial_context::prefixes().mappings().try_fold(
+            oxttl::TurtleSerializer::new().with_base_iri(base.as_str())?,
             |serializer, (prefix, value)| serializer.with_prefix(prefix, value),
         )?;
 
@@ -72,8 +114,11 @@ async fn main() -> Result<ExitCode, Box<dyn std::error::Error>> {
             });
         }
 
-        let mut locked_out = std::io::stdout().lock();
-        locked_out.write_all(
+        let locked_out = std::io::stdout().lock();
+        // yes, stdout is already buffered, but we want to just write big chunks
+        // TTL tends to write lots of short lines which performs pretty poorly with LineWriter
+        let mut out = std::io::BufWriter::new(locked_out);
+        out.write_all(
             format!(
                 "# generated by html2rdf {} at {:.0} from: {}\n",
                 clap::crate_version!(),
@@ -84,16 +129,18 @@ async fn main() -> Result<ExitCode, Box<dyn std::error::Error>> {
         )?;
 
         if args.canonicalize {
-            locked_out.write_all(b"# (output has been canonicalized)\n")?;
+            out.write_all(b"# (output has been canonicalized)\n")?;
         }
 
-        let mut writer = serializer.for_writer(&mut locked_out);
-        for triple in output_graph.iter() {
-            writer.serialize_triple(triple)?;
-        }
+        {
+            let mut writer = serializer.for_writer(&mut out);
+            for triple in &output_graph {
+                writer.serialize_triple(triple)?;
+            }
 
-        writer.finish()?;
-        drop(output_graph);
+            writer.finish()?;
+        }
+        out.flush()?;
     }
 
     Ok(ExitCode::SUCCESS)

html2rdf-cli/src/main.rs

@@ -16,33 +16,45 @@ exclude = [
     "tests/test-suite/test-cases/*/svg*/*",
 ]
 
+[features]
+default = ["html", "http", "xhtml"]
+html = ["dep:scraper"]
+xhtml = ["dep:uppsala"]
+http = ["dep:reqwest"]
+
 [dependencies]
+bergshamra-c14n = "0.3.1"
+bergshamra-xml = "0.3.1"
 curie = "0.1.4"
-derive_more = { version = "2.1.1", features = [
-    "display",
-    "error",
-    "from",
-    "from_str",
-] }
-icu_locale = { version = "2.1.1" }
+derive_more = { workspace = true, features = ["from", "from_str"] }
 indexmap = "2.13.0"
 itertools = "0.14.0"
+jiff = { workspace = true }
 mitsein = "0.9.0"
+ouroboros = "0.18.5"
+oxilangtag = "0.1.5"
 oxiri = { workspace = true }
 oxrdf = { workspace = true }
 oxsdatatypes = "0.2.2"
-rxml_validation = "0.11.0"
-scraper = "0.25.0"
+percent-encoding = "2.3.2"
+reqwest = { workspace = true, optional = true, features = ["blocking"] }
+rxml_validation = "0.12.0"
+scraper = { version = "0.26.0", optional = true, default-features = false, features = ["errors"] }
+thiserror = "2.0.18"
 tracing = "0.1.44"
+uppsala = { version = "0.3.0", optional = true }
 
 [dev-dependencies]
 bolero = "0.13.4"
+indoc = "2.0.7"
 insta = { version = "1.46.3", features = ["glob"] }
 oxrdf = { workspace = true, features = ["rdfc-10"] }
 oxttl = { workspace = true }
 pretty_assertions = "1.4.1"
-pyo3 = "0.28.2"
 rstest = { version = "0.26.1", default-features = false, features = [
     "crate-name",
 ] }
-sha2 = "0.10.9"
+
+[target.'cfg(not(target_os = "windows"))'.dev-dependencies]
+librdfa-wrapper = { workspace = true }
+pyo3 = "0.28.2"

html2rdf/Cargo.toml

@@ -0,0 +1,9 @@
+//! Supplementary algorithms used in RDFa.
+
+mod property_copying;
+mod vocabulary_expansion;
+
+pub use property_copying::property_copying;
+pub use vocabulary_expansion::{
+    OfflineVocabularyResolver, OnlineVocabularyResolver, VocabularyResolver, vocabulary_expansion,
+};