feat(extension-huggingface): support text generation task

- Add the "TextGenerationFilter".
- support huggingface filters could pass "endpoint" keyword arguments when using different filter task.
- add test cases of "TextGenerationFilter".

Author: kokokuo

packages/extension-huggingface/README.md

@@ -27,6 +27,7 @@ VulcanSQL support using Hugging Face tasks by [VulcanSQL Filters](https://vulcan
 
 **⚠️ Caution**: Hugging Face has a [rate limit](https://huggingface.co/docs/api-inference/faq#rate-limits), so it does not allow sending large datasets to the Hugging Face library for processing. Otherwise, using a different Hugging Face model may yield different results or even result in failure.
 
+
 ### Table Question Answering
 
 The [Table Question Answering](https://huggingface.co/docs/api-inference/detailed_parameters#table-question-answering-task) is one of the Natural Language Processing tasks supported by Hugging Face.
@@ -50,7 +51,7 @@ The result will be converted to a JSON string from `huggingface_table_question_a
     "description": "Query Your Data Warehouse Like Exploring One Big View."
   },
   {
-    "repository": "hell-word",
+    "repository": "hello-world",
     "topic": [],
     "description": "Sample repository for testing"
   }
@@ -80,6 +81,7 @@ SELECT {{ data | huggingface_table_question_answering(query="How many repositori
 {% set question = "List display name where gender are female?" %}
 
 -- The "model" keyword argument is optional. If not provided, the default value is 'google/tapas-base-finetuned-wtq'.
+-- The "endpoint" keyword argument is optional. If was provided, use the endpoint instead of the huggingface official URL with provided model if you also gave.
 -- The "wait_for_model" keyword argument is optional. If not provided, the default value is false.
 -- The "use_cache" keyword argument is optional. If not provided, the default value is true.
 SELECT {{ products.value() | huggingface_table_question_answering(query=question, model="microsoft/tapex-base-finetuned-wtq", wait_for_model=true, use_cache=true) }}
@@ -94,3 +96,60 @@ SELECT {{ products.value() | huggingface_table_question_answering(query=question
   }
 ]
 ```
+
+### Text Generation
+
+The [Text Generation](https://huggingface.co/docs/api-inference/detailed_parameters#text-generation-task) is one of the Natural Language Processing tasks supported by Hugging Face.
+
+Using the `huggingface_text_generation` filter. The result will be a string from `huggingface_text_generation`.
+
+**Notice**: The **Text Generation** default model is **gpt2**, If you would like to use the [Meta LLama2](https://huggingface.co/meta-llama) models, you have two method to do:
+
+1. Subscribe to the [Pro Account](https://huggingface.co/pricing#pro). Set the Meta LLama2 model using the `model` keyword argument in `huggingface_text_generation`, e.g: `meta-llama/Llama-2-13b-chat-hf`.
+
+**SQL**:
+
+```sql
+{% set data = [
+  {
+    "rank": 1,
+    "institution": "Massachusetts Institute of Technology (MIT)",
+    "location code":"US",
+    "location":"United States"
+  },
+  {
+    "rank": 2,
+    "institution": "University of Cambridge",
+    "location code":"UK",
+    "location":"United Kingdom"
+  },
+  {
+    "rank": 3,
+    "institution": "Stanford University"
+    "location code":"US",
+    "location":"United States"
+  }
+  -- other universities.....
+] %}
+
+SELECT {{ data | huggingface_text_generation(query="Which university is the top-ranked university?", model="meta-llama/Llama-2-13b-chat-hf") }} as result
+```
+
+**Response**:
+
+```json
+[
+  {
+    "result": "Answer: Based on the provided list, the top-ranked university is Massachusetts Institute of Technology (MIT) with a rank of 1."
+  }
+]
+```
+
+2. Select one of the [Meta LLama2](https://huggingface.co/meta-llama) Models and deploy it to the [Inference Endpoint](https://huggingface.co/inference-endpoints). Set the endpoint URL using the `endpoint` keyword argument in `huggingface_text_generation`.
+
+```sql
+SELECT {{ data | huggingface_text_generation(query="Which university is the top-ranked university?", endpoint='xxx.yyy.zzz.huggingface.cloud') }} as result
+```
+
+
+

packages/extension-huggingface/src/index.ts

@@ -4,7 +4,14 @@ import {
   Runner as HuggingFaceTableQuestionAnsweringFilterRunner,
 } from './lib/filters/tableQuestionAnswering';
 
+import {
+  Builder as TextGenerationFilterBuilder,
+  Runner as TextGenerationFilterRunner,
+} from './lib/filters/textGeneration';
+
 export default [
   HuggingFaceTableQuestionAnsweringFilterBuilder,
   HuggingFaceTableQuestionAnsweringFilterRunner,
+  TextGenerationFilterBuilder,
+  TextGenerationFilterRunner,
 ];

packages/extension-huggingface/src/lib/filters/tableQuestionAnswering.ts

@@ -44,14 +44,14 @@ export const TableQuestionAnsweringFilter: FunctionalFilter = async ({
   // Convert the data result format to table value format
   const table = convertToHuggingFaceTable(value);
   // omit hidden value '__keywords' from args, it generated from nunjucks and not related to HuggingFace.
-  const { query, model, ...inferenceOptions } = omit(args, '__keywords');
+  const { query, model, endpoint, ...inferenceOptions } = omit(args, '__keywords');
   const payload = {
     inputs: { query, table },
   } as TableQuestionAnsweringOptions;
   if (!isEmpty(inferenceOptions)) payload.options = inferenceOptions;
 
   try {
-    const url = getUrl(model);
+    const url = endpoint ? endpoint : getUrl(model);
     const results = await postRequest(url, payload, token);
     // convert to JSON string to make user get the whole result after parsing it in SQL
     return JSON.stringify(results);

packages/extension-huggingface/src/lib/filters/textGeneration.ts

@@ -0,0 +1,94 @@
+import {
+  FunctionalFilter,
+  InternalError,
+  createFilterExtension,
+} from '@vulcan-sql/core';
+import { has, isArray, isEmpty, omit, pick } from 'lodash';
+import {
+  HuggingFaceOptions,
+  InferenceNLPOptions,
+  apiInferenceEndpoint,
+} from '../model';
+import { postRequest } from '../utils';
+
+// More information described the options. See: https://huggingface.co/docs/api-inference/detailed_parameters#text-generation-task
+type TextGenerationOptions = {
+  inputs: string;
+  parameters?: {
+    // Integer to define the top tokens considered within the sample operation to create new text.
+    top_k?: number;
+    // Float to define the tokens that are within the sample operation of text generation. Add tokens in the sample for more probable to least probable until the sum of the probabilities is greater than top_p.
+    top_p?: number;
+    // Default: 0.1. Range: (0.0 - 100.0). The temperature of the sampling operation. 1 means regular sampling, 0 means always take the highest score, 100.0 is getting closer to uniform probability.
+    temperature?: number;
+    // Range: (0.0 - 100.0). The more a token is used within generation the more it is penalized to not be picked in successive generation passes.
+    repetition_penalty?: number;
+    // Default: 250. The amount of new tokens to be generated, this does not include the input length it is a estimate of the size of generated text you want. Each new tokens slows down the request, so look for balance between response times and length of text generated.
+    max_new_tokens?: number;
+    // Range (0-120.0). The amount of time in seconds that the query should take maximum. Network can cause some overhead so it will be a soft limit. Use that in combination with max_new_tokens for best results.
+    max_time?: number;
+    // Default: false. If set to False, the return results will not contain the original query making it easier for prompting.
+    return_full_text?: boolean;
+    // Default: 1. The number of proposition you want to be returned.
+    num_return_sequences?: number;
+    // Whether or not to use sampling, use greedy decoding otherwise.
+    do_sample?: boolean;
+  };
+  options?: InferenceNLPOptions;
+};
+
+/**
+ * Get text generation url. Used meta Llama 2.0 model be default value.
+ * See: https://huggingface.co/docs/api-inference/detailed_parameters#text-generation-task
+ * */
+const getUrl = (model = 'gpt2') => `${apiInferenceEndpoint}/${model}`;
+
+export const TextGenerationFilter: FunctionalFilter = async ({
+  args,
+  value,
+  options,
+}) => {
+  const token = (options as HuggingFaceOptions)?.accessToken;
+  if (!token) throw new InternalError('please given access token');
+
+  if (!isArray(value))
+    throw new InternalError('Input value must be an array of object');
+  if (!(typeof args === 'object') || !has(args, 'query'))
+    throw new InternalError('Must provide "query" keyword argument');
+
+  // Convert the data result to JSON string as question context
+  const context = JSON.stringify(value);
+  // omit hidden value '__keywords' from args, it generated from nunjucks and not related to HuggingFace.
+  const { query, model, endpoint, ...otherArgs } = omit(args, '__keywords');
+  const inferenceOptions = pick(otherArgs, ['use_cache', 'wait_for_model']);
+  const parameters = omit(otherArgs, ['use_cache', 'wait_for_model', 'endpoint']);
+  const payload = {
+    inputs: `Context: ${context}. Question: ${query}}`,
+    parameters: {
+      return_full_text: false,
+      max_new_tokens: 250,
+      temperature: 0.1,
+    }
+  } as TextGenerationOptions;
+  if (!isEmpty(parameters)) payload.parameters = parameters;
+  if (!isEmpty(inferenceOptions)) payload.options = inferenceOptions;
+
+  try {
+    // if not given endpoint, use default HuggingFace inference endpoint
+    const url = endpoint ? endpoint : getUrl(model);
+    const results = await postRequest(url, payload, token);
+    // get the "generated_text" field, and trim leading and trailing white space.
+    return String(results[0]['generated_text']).trim();
+  } catch (error) {
+    throw new InternalError(
+      `Error when sending data to Hugging Face for executing TextGeneration tasks, details: ${
+        (error as Error).message
+      }`
+    );
+  }
+};
+
+export const [Builder, Runner] = createFilterExtension(
+  'huggingface_text_generation',
+  TextGenerationFilter
+);